Java Code Examples for org.apache.pig.ExecType#LOCAL

The following examples show how to use org.apache.pig.ExecType#LOCAL . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFields.java    From logparser with Apache License 2.0 6 votes vote down vote up
@Test
public void fieldsFullBareExampleTest() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Storage.Data data = resetData(pigServer);

    pigServer.registerQuery(
        "Example = " +
        "    LOAD '" + getClass().getResource("/access.log").toString() + "' " +
        "    USING nl.basjes.pig.input.apachehttpdlog.Loader(" +
        "          '" + LOGFORMAT + "', " +
        "          '-map:request.firstline.uri.query.g:HTTP.URI'," +
        "          '-map:request.firstline.uri.query.r:HTTP.URI'," +
        "          '-map:request.firstline.uri.query.s:SCREENRESOLUTION'," +
        "          '-load:nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector:x'" +
        "           );"
    );

    pigServer.registerQuery("STORE Example INTO 'Example' USING mock.Storage();");

    validateExampleResult(data.get("Example"));
}
 
Example 2
Source File: TestMockStorage.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testMockSchema() throws Exception {
  PigServer pigServer = new PigServer(ExecType.LOCAL);
  Data data = resetData(pigServer);

  data.set("foo", "blah:chararray",
      tuple("a"),
      tuple("b"),
      tuple("c")
      );

  pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
  pigServer.registerQuery("B = FOREACH A GENERATE blah as a, blah as b;");
  pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

  assertEquals(schema("a:chararray,b:chararray"), data.getSchema("bar"));

  List<Tuple> out = data.get("bar");
  assertEquals(tuple("a", "a"), out.get(0));
  assertEquals(tuple("b", "b"), out.get(1));
  assertEquals(tuple("c", "c"), out.get(2));
}
 
Example 3
Source File: TestPigServer.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test(expected = RuntimeException.class)
public void testLocationStrictCheck() throws ExecException, IOException {
    Properties properties = PropertiesUtil.loadDefaultProperties();
    properties.setProperty("pig.location.check.strict", "true");
    PigServer pigServer = new PigServer(ExecType.LOCAL, properties);
    Data data = resetData(pigServer);

    data.set("foo",
            tuple("a", 1, "b"),
            tuple("b", 2, "c"),
            tuple("c", 3, "d"));

    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("B = order A by f1,f2,f3 DESC;");
    pigServer.registerQuery("C = order A by f1,f2,f3;");
    // Storing to same location 'bar' should throw a RuntimeException
    pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");
    pigServer.registerQuery("STORE C INTO 'bar' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(tuple("a", 1, "b"), out.get(0));
    assertEquals(tuple("b", 2, "c"), out.get(1));
    assertEquals(tuple("c", 3, "d"), out.get(2));
}
 
Example 4
Source File: TestAssert.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * Verify that alias is not assignable to the ASSERT operator
 * @throws Exception
 */
@Test(expected=FrontendException.class)
public void testNegativeWithAlias() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("foo",
            tuple(1),
            tuple(2),
            tuple(3)
            );
    try {
        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = ASSERT A BY i > 1 , 'i should be greater than 1';");
    }
    catch (FrontendException fe) {
        Util.checkMessageInException(fe, "Syntax error, unexpected symbol at or near 'B'");
        throw fe;
    }
}
 
Example 5
Source File: TestPigServer.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
// See PIG-3967
public void testGruntValidation() throws IOException {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("foo",
            tuple("a", 1, "b"),
            tuple("b", 2, "c"),
            tuple("c", 3, "d"));

    pigServer.setValidateEachStatement(true);
    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("store A into '" + Util.generateURI(tempDir.toString(), pigServer.getPigContext()) + "/testGruntValidation1';");
    pigServer.registerQuery("B = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("store B into '" + Util.generateURI(tempDir.toString(), pigServer.getPigContext()) + "/testGruntValidation2';"); // This should pass
    boolean validationExceptionCaptured = false;
    try {
        // This should fail due to output validation
        pigServer.registerQuery("store A into '" + Util.generateURI(tempDir.toString(),pigServer.getPigContext()) + "/testGruntValidation1';");
    } catch (FrontendException e) {
        validationExceptionCaptured = true;
    }

    assertTrue(validationExceptionCaptured);
}
 
Example 6
Source File: TestUnion.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testCastingAfterUnion() throws Exception {
    File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"aaa\t111"});
    File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"bbb\t222"});

    PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
    ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a,b);");
    ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' as (a,b);");
    ps.registerQuery("C = union A,B;");
    ps.registerQuery("D = foreach C generate (chararray)a as a,(int)b as b;");

    Schema dumpSchema = ps.dumpSchema("D");
    Schema expected = new Schema ();
    expected.add(new Schema.FieldSchema("a", DataType.CHARARRAY));
    expected.add(new Schema.FieldSchema("b", DataType.INTEGER));
    assertEquals(expected, dumpSchema);

    Iterator<Tuple> itr = ps.openIterator("D");
    int recordCount = 0;
    while(itr.next() != null)
        ++recordCount;
    assertEquals(2, recordCount);

}
 
Example 7
Source File: TestBuiltInBagToTupleOrString.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testPigScriptrForBagToStringUDF() throws Exception {
	PigServer pigServer = new PigServer(ExecType.LOCAL);
	Data data = resetData(pigServer);

	data.set("foo", "myBag:bag{t:(l:chararray)}",
			tuple(bag(tuple("a"), tuple("b"), tuple("c"))));
	pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
	pigServer.registerQuery("B = FOREACH A GENERATE BagToString(myBag) as myBag;");
    pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

    pigServer.registerQuery("C = FOREACH A GENERATE BagToString(myBag, '==') as myBag;");
    pigServer.registerQuery("STORE C INTO 'baz' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(schema("myBag:chararray"), data.getSchema("bar"));
    assertEquals(tuple("a_b_c"), out.get(0));

    out = data.get("baz");
    assertEquals(tuple("a==b==c"), out.get(0));
}
 
Example 8
Source File: TestUnionOnSchema.java    From spork with Apache License 2.0 6 votes vote down vote up
private void checkSchemaEx(String query, String expectedErr) throws IOException {
    PigServer pig = new PigServer(ExecType.LOCAL);

    boolean foundEx = false;
    try{
        Util.registerMultiLineQuery(pig, query);
        pig.dumpSchema("u");
    }catch(FrontendException e){
        PigException pigEx = LogUtils.getPigException(e);
        foundEx = true;
        if(!pigEx.getMessage().contains(expectedErr)){
            String msg = "Expected exception message matching '" 
                + expectedErr + "' but got '" + pigEx.getMessage() + "'" ;
            fail(msg);
        }
    }
    
    if(!foundEx)
        fail("No exception thrown. Exception is expected.");
    
   
}
 
Example 9
Source File: TestPigServer.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testPigTempDir() throws Throwable {
    Properties properties = PropertiesUtil.loadDefaultProperties();
    File pigTempDir = new File(tempDir, FILE_SEPARATOR + "tmp" + FILE_SEPARATOR + "test");
    properties.put("pig.temp.dir", pigTempDir.getPath());
    PigContext pigContext=new PigContext(ExecType.LOCAL, properties);
    pigContext.connect();
    FileLocalizer.setInitialized(false);

    String tempPath= FileLocalizer.getTemporaryPath(pigContext).toString();
    Path path = new Path(tempPath);
    assertTrue(tempPath.startsWith(pigTempDir.toURI().toString()));

    FileSystem fs = FileSystem.get(path.toUri(),
            ConfigurationUtil.toConfiguration(pigContext.getProperties()));
    FileStatus status = fs.getFileStatus(path.getParent());
    // Temporary root dir should have 700 as permission
    assertEquals("rwx------", status.getPermission().toString());
    pigTempDir.delete();
    FileLocalizer.setInitialized(false);
}
 
Example 10
Source File: TestBuiltInBagToTupleOrString.java    From spork with Apache License 2.0 6 votes vote down vote up
@Test
public void testPigScriptMultipleElmementsPerTupleForBagToStringUDF() throws Exception {
	PigServer pigServer = new PigServer(ExecType.LOCAL);
	Data data = resetData(pigServer);

	data.set("foo", "myBag:bag{t:(l:chararray)}",
			tuple(bag(tuple("a", "b"), tuple("c", "d"), tuple("e", "f"))));
	pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
	pigServer.registerQuery("B = FOREACH A GENERATE BagToString(myBag) as myBag;");
	pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

	pigServer.registerQuery("C = FOREACH A GENERATE BagToString(myBag, '^') as myBag;");
	pigServer.registerQuery("STORE C INTO 'baz' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(tuple("a_b_c_d_e_f"), out.get(0));

    out = data.get("baz");
    assertEquals(tuple("a^b^c^d^e^f"), out.get(0));
}
 
Example 11
Source File: TestPigStats.java    From spork with Apache License 2.0 5 votes vote down vote up
@Test
public void testPigStatsGetList() {
    File outputFile = null;
    try {
        String filename = this.getClass().getSimpleName() + "_" + "testPigStatsGetList";
        outputFile = File.createTempFile(filename, ".out");
        String filePath = outputFile.getAbsolutePath();
        outputFile.delete();
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        pigServer.registerQuery("a = load 'test/org/apache/pig/test/data/passwd';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
        pigServer.registerQuery("d = group c by cnt;");
        pigServer.registerQuery("e = foreach d generate group;");
        ExecJob job = pigServer.store("e", filePath);
        JobGraph jobGraph = job.getStatistics().getJobGraph();
        assertEquals(2, jobGraph.getJobList().size());

    } catch (IOException e) {
        LOG.error("IOException while creating file ", e);
        fail("Encountered IOException");
    } finally {
        if (outputFile != null) {
            // delete the directory before returning
            deleteDirectory(outputFile);
        }
    }
}
 
Example 12
Source File: TestFixedWidthStorer.java    From spork with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws IOException {
    pig = new PigServer(ExecType.LOCAL);

    Util.deleteDirectory(new File(dataDir));
    try {
        pig.mkdirs(dataDir);
    } catch (IOException e) {};
}
 
Example 13
Source File: TestUnionOnSchema.java    From spork with Apache License 2.0 5 votes vote down vote up
/**
 * Test UNION ONSCHEMA where a common column has additional 'namespace' part
 *  in the column name in both the inputs
 * @throws IOException
 * @throws ParserException
 */
@Test
public void testUnionOnSchemaScopedColumnNameBothInp1() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL);
    String query = 
    "  l1 = load '" + INP_FILE_2NUMS + "' as (i : int, j : int); " 
    + "g1 = group l1 by i; "
    + "f1 = foreach g1 generate group as gkey, flatten(l1); "
    + "l2 = load '" + INP_FILE_2NUMS + "' as (i : int, x : chararray); " 
    + "g2 = group l2 by i; "
    + "f2 = foreach g2 generate group as gkey, flatten(l2); "
    + "u = union onschema f1, f2; " ; 
    Util.registerMultiLineQuery(pig, query);
    
    Schema sch = pig.dumpSchema("u");
    Schema expectedSch = 
        Utils.getSchemaFromString("gkey: int, l1::i: int, l1::j: int, l2::i: int, l2::x: chararray");
    assertEquals("Checking expected schema",sch, expectedSch);

    Iterator<Tuple> it = pig.openIterator("u");
    List<Tuple> expectedRes = 
        Util.getTuplesFromConstantTupleStrings(
                new String[] {
                        "(1,1,2,null,null)",
                        "(5,5,3,null,null)",
                        "(1,null,null,1,'2')",
                        "(5,null,null,5,'3')"
                });
    Util.checkQueryOutputsAfterSort(it, expectedRes);
}
 
Example 14
Source File: TestMultiQueryBasic.java    From spork with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
    Util.copyFromLocalToLocal(
            "test/org/apache/pig/test/data/passwd", "passwd");
    Util.copyFromLocalToLocal(
            "test/org/apache/pig/test/data/passwd2", "passwd2");
    Properties props = new Properties();
    props.setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, ""+true);
    myPig = new PigServer(ExecType.LOCAL, props);
}
 
Example 15
Source File: VespaQueryTest.java    From vespa with Apache License 2.0 5 votes vote down vote up
private PigServer setup(String script, String endpoint) throws Exception {
    Configuration conf = new HdfsConfiguration();
    Map<String, String> parameters = new HashMap<>();
    parameters.put("ENDPOINT", endpoint);

    PigServer ps = new PigServer(ExecType.LOCAL, conf);
    ps.setBatchOn();
    ps.registerScript(script, parameters);

    return ps;
}
 
Example 16
Source File: TestUDF.java    From spork with Apache License 2.0 5 votes vote down vote up
@Test
public void testUDFReturnMap_LocalMode() throws Exception {
    PigServer pig = new PigServer(ExecType.LOCAL);
    pig.registerScript(TempScriptFile.getAbsolutePath());

    Iterator<Tuple> iterator = pig.openIterator("B");
    while (iterator.hasNext()) {
        Tuple tuple = iterator.next();
        @SuppressWarnings("unchecked")
        Map<Object, Object> result = (Map<Object, Object>) tuple.get(0);
        assertEquals(result, MyUDFReturnMap.map);
    }
}
 
Example 17
Source File: TestPigScriptParser.java    From spork with Apache License 2.0 5 votes vote down vote up
@Test
public void testSplitWithNotEvalCondition() throws Exception {
    String defineQ = "define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');";
    String defineL = "a = load 'nosuchfile' " +
            " using PigStorage() as (source : chararray);";
    String defineSplit = "SPLIT a INTO a1 IF (minelogs(source) > 0 ), a2 IF (NOT (minelogs(source)>0));";//    (NOT ( minelogs(source) ) > 0);";
    PigServer ps = new PigServer(ExecType.LOCAL);
    ps.registerQuery(defineQ);
    ps.registerQuery(defineL);
    ps.registerQuery(defineSplit);
}
 
Example 18
Source File: TestUnion.java    From spork with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
    pigServer = new PigServer(ExecType.LOCAL, new Properties());
    pc = pigServer.getPigContext();
    pc.connect();
    GenPhyOp.setPc(pc);
    POLoad ld1 = GenPhyOp.topLoadOp();
    String curDir = System.getProperty("user.dir");
    String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
    FileSpec fSpec = new FileSpec(Util.generateURI(inpDir + "passwd", pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
    ld1.setLFile(fSpec);

    POLoad ld2 = GenPhyOp.topLoadOp();
    ld2.setLFile(fSpec);

    POFilter fl1 = GenPhyOp.topFilterOpWithProj(1, 50, GenPhyOp.LTE);

    POFilter fl2 = GenPhyOp.topFilterOpWithProj(1, 50, GenPhyOp.GT);

    int[] flds = {0,2};
    Tuple sample = new DefaultTuple();
    sample.append(new String("S"));
    sample.append(new String("x"));
    sample.append(new Integer("10"));
    sample.append(new Integer("20"));
    sample.append(new String("S"));
    sample.append(new String("x"));
    sample.append(new String("S"));
    sample.append(new String("x"));

    POForEach fe1 = GenPhyOp.topForEachOPWithPlan(flds , sample);

    POForEach fe2 = GenPhyOp.topForEachOPWithPlan(flds , sample);

    sp = GenPhyOp.topUnionOp();

    PhysicalPlan plan = new PhysicalPlan();

    plan.add(ld1);
    plan.add(ld2);
    plan.add(fl1);
    plan.add(fl2);
    plan.add(fe1);
    plan.add(fe2);
    plan.add(sp);

    plan.connect(ld1, fe1);
    plan.connect(fe1, fl1);
    plan.connect(ld2, fe2);
    plan.connect(fe2, fl2);
    plan.connect(fl1, sp);
    plan.connect(fl2, sp);

    /*PlanPrinter ppp = new PlanPrinter(plan);
    ppp.visit();*/


    POLoad ld3 = GenPhyOp.topLoadOp();
    ld3.setLFile(fSpec);
    DataBag fullBag = DefaultBagFactory.getInstance().newDefaultBag();
    Tuple t=null;
    for(Result res=ld3.getNextTuple();res.returnStatus!=POStatus.STATUS_EOP;res=ld3.getNextTuple()){
        fullBag.add((Tuple)res.result);
    }

    int[] fields = {0,2};
    expBag = TestHelper.projectBag(fullBag, fields);
}
 
Example 19
Source File: TestLocal2.java    From spork with Apache License 2.0 4 votes vote down vote up
public TestLocal2() throws Throwable {
    pig = new PigServer(ExecType.LOCAL) ;
}
 
Example 20
Source File: TestGruntParser.java    From spork with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws IOException {
    pig = new PigServer(ExecType.LOCAL);
}