org.apache.pig.FuncSpec Java Examples

The following examples show how to use org.apache.pig.FuncSpec. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: TestPOUserFunc.java From spork with Apache License 2.0

6 votes

public void userFuncArity(DataBag input ) throws ExecException {
	String funcSpec = ARITY.class.getName() + "()";
	PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
	List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
	inputs.add(read);
	POUserFunc userFunc = new POUserFunc(new OperatorKey("", r.nextLong()),
			-1, inputs, new FuncSpec(funcSpec));
	Result res = new Result();
	Integer i = null;
	res = userFunc.getNextInteger();
	while (res.returnStatus != POStatus.STATUS_EOP) {
		// System.out.println(res.result);
		int result = (Integer) res.result;
		assertEquals(2, result);
		res = userFunc.getNextInteger();
	}
}

Example #2

Source File: GenPhyOp.java From spork with Apache License 2.0

6 votes

public static POForEach topForEachOPWithUDF(List<String> clsName) throws PlanException{
    List<PhysicalPlan> ep4s = new ArrayList<PhysicalPlan>();
    List<Boolean> flattened3 = new ArrayList<Boolean>();
    for (String string : clsName) {
        PhysicalPlan ep4 = new PhysicalPlan();
        POProject prjStar4 = new POProject(new OperatorKey("", r.nextLong()));
        prjStar4.setResultType(DataType.TUPLE);
        prjStar4.setStar(true);
        ep4.add(prjStar4);
        
        List ufInps = new ArrayList();
        ufInps.add(prjStar4);
        POUserFunc uf = new POUserFunc(new OperatorKey("", r.nextLong()), -1, ufInps, new FuncSpec(string));
        ep4.add(uf);
        ep4.connect(prjStar4, uf);
        ep4s.add(ep4);
        flattened3.add(false);
    }
    
    POForEach fe3 = new POForEach(new OperatorKey("", r.nextLong()), 1,
        ep4s, flattened3);
    fe3.setResultType(DataType.TUPLE);
    return fe3;
}

Example #3

Source File: TestLOLoadDeterminedSchema.java From spork with Apache License 2.0

6 votes

/**
 * Loads a test file using ScriptSchemaTestLoader with a user defined schema
 * a,b,c.<br/>
 * Then tests the the ScriptSchemaTestLoader found the schema.
 *
 * @throws IOException
 */
@Test
public void testDeterminedSchema() throws IOException {
    FuncSpec funcSpec = new FuncSpec(ScriptSchemaTestLoader.class.getName()
            + "()");

    server.registerFunction(ScriptSchemaTestLoader.class.getName(),
            funcSpec);

    server.registerQuery("a = LOAD '" + Util.encodeEscape(inputFile.getAbsolutePath())
            + "' using " + ScriptSchemaTestLoader.class.getName()
            + "() as (a, b, c) ;");

    server.openIterator("a");

    Schema scriptSchema = ScriptSchemaTestLoader.getScriptSchema();

    assertNotNull(scriptSchema);
    assertEquals(3, scriptSchema.size());

    assertNotNull(scriptSchema.getField("a"));
    assertNotNull(scriptSchema.getField("b"));
    assertNotNull(scriptSchema.getField("c"));
}

Example #4

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testLoadEqualityDifferentFunctionNames() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different function names in FuncSpec
    LOLoad load4 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load4);

    assertFalse(load1.isEqual(load4));
}

Example #5

Source File: TestHiveColumnarLoader.java From spork with Apache License 2.0

6 votes

@Test
public void testDatePartitionedFiles() throws IOException {
    int count = 0;

    String funcSpecString = "org.apache.pig.piggybank.storage.HiveColumnarLoader('f1 string,f2 string,f3 string'"
            + ", '" + startingDate + ":" + endingDate + "')";

    System.out.println(funcSpecString);

    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
            new FuncSpec(funcSpecString));

    server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
            + funcSpecString + ";");
    Iterator<Tuple> result = server.openIterator("a");

    while ((result.next()) != null) {
        count++;
    }

    Assert.assertEquals(datePartitionedRowCount, count);
}

Example #6

Source File: STRSPLIT.java From spork with Apache License 2.0

6 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY)); 
    
    Schema s1 = new Schema();
    s1.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s1.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    
    Schema s2 = new Schema();
    s2.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s2.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s2.add(new Schema.FieldSchema(null, DataType.INTEGER));
    
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    funcList.add(new FuncSpec(this.getClass().getName(), s1));
    funcList.add(new FuncSpec(this.getClass().getName(), s2));
    return funcList;
}

Example #7

Source File: TestMRJobStats.java From spork with Apache License 2.0

6 votes

private static POStore createPOStoreForFileBasedSystem(long size, StoreFuncInterface storeFunc,
        Configuration conf) throws Exception {

    File file = File.createTempFile("tempFile", ".tmp");
    file.deleteOnExit();
    RandomAccessFile f = new RandomAccessFile(file, "rw");
    f.setLength(size);
    f.close();

    storeFunc.setStoreLocation(file.getAbsolutePath(), new Job(conf));
    FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName());
    POStore poStore = new POStore(new OperatorKey());
    poStore.setSFile(new FileSpec(file.getAbsolutePath(), funcSpec));
    poStore.setStoreFunc(storeFunc);
    poStore.setUp();

    return poStore;
}

Example #8

Source File: CastExpression.java From spork with Apache License 2.0

6 votes

@Override
public LogicalExpression deepCopy(LogicalExpressionPlan lgExpPlan) throws FrontendException {
    LogicalExpression copy = new CastExpression(
            lgExpPlan,
            this.getExpression().deepCopy(lgExpPlan),
            castSchema.deepCopy());
    try {
        FuncSpec origFuncSpec = this.getFuncSpec();
        if (origFuncSpec != null ) {
            ((CastExpression)copy).setFuncSpec(origFuncSpec.clone());
        }
    } catch(CloneNotSupportedException e) {
        e.printStackTrace(); 
    }
    copy.setLocation( new SourceLocation( location ) );
    return copy;
}

Example #9

Source File: TestHiveColumnarLoader.java From spork with Apache License 2.0

6 votes

@Test
public void test1DayDatePartitionedFiles() throws IOException {
    int count = 0;

    String funcSpecString = "org.apache.pig.piggybank.storage.HiveColumnarLoader('f1 string,f2 string,f3 string'"
            + ", '" + startingDate + ":" + startingDate + "')";

    System.out.println(funcSpecString);

    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
            new FuncSpec(funcSpecString));

    server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
            + funcSpecString + ";");
    Iterator<Tuple> result = server.openIterator("a");

    while ((result.next()) != null) {
        count++;
    }

    Assert.assertEquals(50, count);
}

Example #10

Source File: SecondsBetween.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema();
    s.add(new Schema.FieldSchema(null, DataType.DATETIME));
    s.add(new Schema.FieldSchema(null, DataType.DATETIME));
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    return funcList;
}

Example #11

Source File: LTRIM.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY))));

    return funcList;
}

Example #12

Source File: LAST_INDEX_OF.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema();
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    return funcList;
}

Example #13

Source File: DaysBetween.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema();
    s.add(new Schema.FieldSchema(null, DataType.DATETIME));
    s.add(new Schema.FieldSchema(null, DataType.DATETIME));
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    return funcList;
}

Example #14

Source File: AddDuration.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema();
    s.add(new Schema.FieldSchema(null, DataType.DATETIME));
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    return funcList;
}

Example #15

Source File: MAX.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
    funcList.add(new FuncSpec(DoubleMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
    funcList.add(new FuncSpec(FloatMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
    funcList.add(new FuncSpec(IntMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
    funcList.add(new FuncSpec(LongMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
    funcList.add(new FuncSpec(StringMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.CHARARRAY)));
    funcList.add(new FuncSpec(DateTimeMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DATETIME)));
    funcList.add(new FuncSpec(BigDecimalMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BIGDECIMAL)));
    funcList.add(new FuncSpec(BigIntegerMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BIGINTEGER)));
    return funcList;
}

Example #16

Source File: TezCompiler.java From spork with Apache License 2.0

5 votes

private void fixScalar() throws VisitorException, PlanException {
    // Mapping POStore to POValueOuptut
    Map<POStore, POValueOutputTez> storeSeen = new HashMap<POStore, POValueOutputTez>();

    for (TezOperator tezOp : tezPlan) {
        List<POUserFunc> userFuncs = PlanHelper.getPhysicalOperators(tezOp.plan, POUserFunc.class);
        for (POUserFunc userFunc : userFuncs) {
            if (userFunc.getReferencedOperator()!=null) {  // Scalar
                POStore store = (POStore)userFunc.getReferencedOperator();

                TezOperator from = phyToTezOpMap.get(store);

                FuncSpec newSpec = new FuncSpec(ReadScalarsTez.class.getName(), from.getOperatorKey().toString());
                userFunc.setFuncSpec(newSpec);

                if (storeSeen.containsKey(store)) {
                    storeSeen.get(store).addOutputKey(tezOp.getOperatorKey().toString());
                } else {
                    POValueOutputTez output = new POValueOutputTez(OperatorKey.genOpKey(scope));
                    output.addOutputKey(tezOp.getOperatorKey().toString());
                    from.plan.remove(from.plan.getOperator(store.getOperatorKey()));
                    from.plan.addAsLeaf(output);
                    storeSeen.put(store, output);

                    //Remove unused store filename
                    userFunc.getInputs().remove(1);
                }

                TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, from, tezOp);
                //TODO shared edge once support is available in Tez
                TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.BROADCAST);
            }
        }
    }
}

Example #17

Source File: GetWeekYear.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.DATETIME))));

    return funcList;
}

Example #18

Source File: GetSecond.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.DATETIME))));

    return funcList;
}

Example #19

Source File: DiffDate.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema();
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    return funcList;
}

Example #20

Source File: GetDay.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.DATETIME))));

    return funcList;
}

Example #21

Source File: DateExtractor.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), 
        new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY))));

    return funcList;
}

Example #22

Source File: ROUND_TO.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();

    Schema s_bty_2 = new Schema();
    s_bty_2.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
    s_bty_2.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
    Schema s_bty_3 = new Schema();
    s_bty_3.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
    s_bty_3.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
    s_bty_3.add(new Schema.FieldSchema(null, DataType.INTEGER));

    Schema s_dbl_2 = new Schema();
    s_dbl_2.add(new Schema.FieldSchema(null, DataType.DOUBLE));
    s_dbl_2.add(new Schema.FieldSchema(null, DataType.INTEGER));
    Schema s_dbl_3 = new Schema();
    s_dbl_3.add(new Schema.FieldSchema(null, DataType.DOUBLE));
    s_dbl_3.add(new Schema.FieldSchema(null, DataType.INTEGER));
    s_dbl_3.add(new Schema.FieldSchema(null, DataType.INTEGER));

    Schema s_flt_2 = new Schema();
    s_flt_2.add(new Schema.FieldSchema(null, DataType.FLOAT));
    s_flt_2.add(new Schema.FieldSchema(null, DataType.INTEGER));
    Schema s_flt_3 = new Schema();
    s_flt_3.add(new Schema.FieldSchema(null, DataType.FLOAT));
    s_flt_3.add(new Schema.FieldSchema(null, DataType.INTEGER));
    s_flt_3.add(new Schema.FieldSchema(null, DataType.INTEGER));

    funcList.add(new FuncSpec(this.getClass().getName(),     s_bty_2));
    funcList.add(new FuncSpec(this.getClass().getName(),     s_bty_3));
    funcList.add(new FuncSpec(DoubleRoundTo.class.getName(), s_dbl_2));
    funcList.add(new FuncSpec(DoubleRoundTo.class.getName(), s_dbl_3));
    funcList.add(new FuncSpec(FloatRoundTo.class.getName(),  s_flt_2));
    funcList.add(new FuncSpec(FloatRoundTo.class.getName(),  s_flt_3));

    return funcList;
}

Example #23

Source File: LogicalPlanBuilder.java From spork with Apache License 2.0

5 votes

Operator buildNestedSortOp(SourceLocation loc, LOSort op, LogicalPlan plan, String alias, Operator inputOp,
        List<LogicalExpressionPlan> plans,
        List<Boolean> ascFlags, FuncSpec fs) {
    op.setSortColPlans( plans );
    if (ascFlags.isEmpty()) {
        for (int i=0;i<plans.size();i++)
            ascFlags.add(true);
    }
    op.setAscendingCols( ascFlags );
    op.setUserFunc( fs );
    buildNestedOp( loc, plan, op, alias, inputOp );
    return op;
}

Example #24

Source File: SearchEngineExtractor.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), 
        new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY))));

    return funcList;
}

Example #25

Source File: TestFRJoin.java From spork with Apache License 2.0

5 votes

private void setUpHashTable() throws IOException {
    FileSpec replFile = new FileSpec(repl, new FuncSpec(PigStorage.class.getName() + "()"));
    POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
    PigContext pc = new PigContext(ExecType.MAPREDUCE, PigMapReduce.sJobConfInternal.get());
    pc.connect();

    ld.setPc(pc);
    for (Result res = ld.getNextTuple(); res.returnStatus != POStatus.STATUS_EOP; res = ld
            .getNextTuple()) {
        Tuple tup = (Tuple)res.result;
        LoadFunc lf = ((LoadFunc)PigContext.instantiateFuncFromSpec(ld.getLFile().getFuncSpec()));
        String key = lf.getLoadCaster().bytesToCharArray(
                ((DataByteArray)tup.get(keyField)).get());
        Tuple csttup = TupleFactory.getInstance().newTuple(2);
        csttup.set(0, key);
        csttup.set(1, lf.getLoadCaster().bytesToInteger(((DataByteArray)tup.get(1)).get()));
        DataBag vals = null;
        if (replTbl.containsKey(key)) {
            vals = replTbl.get(key);
        }
        else {
            vals = BagFactory.getInstance().newDefaultBag();
            replTbl.put(key, vals);
        }
        vals.add(csttup);
    }
}

Example #26

Source File: LogicalPlanBuilder.java From spork with Apache License 2.0

5 votes

String buildSampleOp(SourceLocation loc, LOFilter filter, String alias, String inputAlias,
        LogicalExpressionPlan samplePlan, LogicalExpression expr)
                throws ParserValidationException {

    UserFuncExpression udf = new UserFuncExpression( samplePlan, new FuncSpec( RANDOM.class.getName() ) );
    new LessThanExpression( samplePlan, udf, expr );
    return buildFilterOp( loc, filter, alias, inputAlias, samplePlan );
}

Example #27

Source File: LogicalPlanBuilder.java From spork with Apache License 2.0

5 votes

String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
        SourceLocation valLoc)
                throws ParserValidationException {

    LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
    //  Generate a filter condition.
    LogicalExpression konst = new ConstantExpression( filterPlan, value);
    konst.setLocation( valLoc );
    UserFuncExpression udf = new UserFuncExpression( filterPlan, new FuncSpec( RANDOM.class.getName() ) );
    new LessThanExpression( filterPlan, udf, konst );
    LOFilter filter = new LOFilter( plan, true );
    return buildFilterOp( loc, filter, alias, inputAlias, filterPlan );
}

Example #28

Source File: LogicalPlanBuilder.java From spork with Apache License 2.0

5 votes

void defineFunction(String alias, FuncSpec fs) {
    try {
        filter.validate(PigCommandFilter.Command.DEFINE);
    } catch (FrontendException e) {
        throw new RuntimeException(e);
    }
    pigContext.registerFunction( alias, fs );
}

Example #29

Source File: MAX.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Util.addToFunctionList(funcList, this.getClass().getName(), DataType.BYTEARRAY);
    Util.addToFunctionList(funcList, DoubleMax.class.getName(), DataType.DOUBLE);
    Util.addToFunctionList(funcList, FloatMax.class.getName(), DataType.FLOAT);
    Util.addToFunctionList(funcList, IntMax.class.getName(), DataType.INTEGER);
    Util.addToFunctionList(funcList, LongMax.class.getName(), DataType.LONG);

    return funcList;
}

Example #30

Source File: RegexExtract.java From spork with Apache License 2.0

5 votes

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema();
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s.add(new Schema.FieldSchema(null, DataType.INTEGER));
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    return funcList;
}