Java Code Examples for org.apache.pig.PigServer#getPigContext()

The following examples show how to use org.apache.pig.PigServer#getPigContext() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestGrunt.java From spork with Apache License 2.0

6 votes

@Test
public void testIllustrate() throws Throwable {
    Assume.assumeTrue("Skip this test for TEZ. See PIG-3993", Util.isMapredExecType(cluster.getExecType()));
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd =
        "rmf bla;"
        +"a = load '"
        + Util.generateURI("file:test/org/apache/pig/test/data/passwd", context) + "';"
        +"e = group a by $0;"
        +"f = foreach e generate group, COUNT($1);"
        +"store f into 'bla';"
        +"f1 = load 'bla' as (f:chararray);"
        +"g = order f1 by $0;"
        +"illustrate g;";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 2

Source File: TestGrunt.java From spork with Apache License 2.0

6 votes

@Test
public void testExecStatmentNested() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();
    boolean caught = false;

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); exec "
            + basedir + "/testsubnested_exec.pig; explain bar";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    try {
        grunt.exec();
    } catch (Exception e) {
        caught = true;
        assertTrue(e.getMessage().contains("alias bar"));
    }
    assertTrue(caught);
}

Example 3

Source File: TestGrunt.java From spork with Apache License 2.0

6 votes

@Test
public void testRegisterWithQuotes() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();
    String jarName = Util.findPigJarName();

    String strCmd = "register '" + jarName + "'\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
    assertEquals(context.extraJars+ " of size 1", 1, context.extraJars.size());
    assertTrue(context.extraJars.get(0)+" ends with /" + jarName, context.extraJars.get(0).toString().endsWith("/" + jarName));
}

Example 4

Source File: TestLoad.java From spork with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
@Test
public void testCommaSeparatedString3() throws Exception {
    PigServer pig = servers[0];
    pc = pig.getPigContext();
    boolean noConversionExpected = true;
    checkLoadPath("hdfs:/tmp/test,hdfs:/tmp/test2,hdfs:/tmp/test3",
            "hdfs:/tmp/test,hdfs:/tmp/test2,hdfs:/tmp/test3", noConversionExpected );

    // check if a location 'hdfs:<abs path>,hdfs:<abs path>' can actually be
    // read using PigStorage
    String[] inputFileNames = new String[] {
            "/tmp/TestLoad-testCommaSeparatedString3-input1.txt",
            "/tmp/TestLoad-testCommaSeparatedString3-input2.txt"};
    String inputString = "hdfs:" + inputFileNames[0] + ",hdfs:" +
    inputFileNames[1];
    testLoadingMultipleFiles(inputFileNames, inputString);

}

Example 5

Source File: TestGrunt.java From spork with Apache License 2.0

6 votes

@Test
public void testParsingAsGenerateInForeachBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
            + "b = group a by foo; c = foreach b {generate "
            + "{(1, '1', 0.4f),(2, '2', 0.45)} "
            + "as b: bag{t:(i: int, cease:chararray, degenerate: double)}, "
            + "SUM(a.fast) as fast, a.regenerate as degenerated;};\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 6

Source File: TestGrunt.java From spork with Apache License 2.0

6 votes

@Test
public void testInvalidParam() throws Throwable {
    PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd =
        "run -param -param;";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    boolean caught = false;
    try {
        grunt.exec();
    } catch (ParseException e) {
        caught = true;
        assertTrue(e.getMessage().contains("Encountered"));
    }
    assertTrue(caught);
}

Example 7

Source File: TestGrunt.java From spork with Apache License 2.0

5 votes

@Test
public void testParsingGenerateInForeachWithOutBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
            + "b = group a by foo; c = foreach b generate a.regenerate;\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 8

Source File: TestJobControlCompiler.java From spork with Apache License 2.0

5 votes

@Test
public void testAddArchiveToDistributedCache() throws IOException {
    final File textFile = File.createTempFile("file", ".txt");
    textFile.deleteOnExit();

    final List<File> zipArchives = createFiles(".zip");
    zipArchives.add(textFile);
    final List<File> tarArchives = createFiles(".tgz", ".tar.gz", ".tar");

    final PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
    final PigContext pigContext = pigServer.getPigContext();
    pigContext.connect();
    pigContext.getProperties().put("pig.streaming.ship.files",
            StringUtils.join(zipArchives, ","));
    pigContext.getProperties().put("pig.streaming.cache.files",
            StringUtils.join(tarArchives, ","));

    final JobConf jobConf = compileTestJob(pigContext, CONF);

    URI[] uris = DistributedCache.getCacheFiles(jobConf);
    int sizeTxt = 0;
    for (int i = 0; i < uris.length; i++) {
        if (uris[i].toString().endsWith(".txt")) {
            sizeTxt++;
        }
    }
    Assert.assertTrue(sizeTxt == 1);
    assertFilesInDistributedCache(
            DistributedCache.getCacheArchives(jobConf), 4, ".zip", ".tgz",
            ".tar.gz", ".tar");
}

Example 9

Source File: TestLoad.java From spork with Apache License 2.0

5 votes

@Test
public void testCommaSeparatedString2() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        checkLoadPath("t?s*,test","/tmp/t?s*,/tmp/test");
    }
}

Example 10

Source File: TestLoad.java From spork with Apache License 2.0

5 votes

@Test
public void testCommaSeparatedString4() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        checkLoadPath("usr/pig/{a,c},usr/pig/b","/tmp/usr/pig/{a,c},/tmp/usr/pig/b");
    }
}

Example 11

Source File: TestGrunt.java From spork with Apache License 2.0

5 votes

@Test
public void testBagConstant() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'input1'; b = foreach a generate {(1, '1', 0.4f),(2, '2', 0.45)};\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 12

Source File: TestLoad.java From spork with Apache License 2.0

5 votes

@Test
public void testCommaSeparatedString6() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        checkLoadPath("usr/pig/{a,c},/usr/pig/b","/tmp/usr/pig/{a,c},/usr/pig/b");
    }
}

Example 13

Source File: TestLoad.java From spork with Apache License 2.0

5 votes

@Test
public void testLoadRemoteAbs() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        boolean noConversionExpected = true;
        checkLoadPath("/tmp/test","/tmp/test", noConversionExpected);
    }
}

Example 14

Source File: TestStore.java From spork with Apache License 2.0

5 votes

@Before
public void setUp() throws Exception {
    pig = new PigServer(cluster.getExecType(), cluster.getProperties());
    pc = pig.getPigContext();
    inputFileName = TESTDIR + "/TestStore-" + new Random().nextLong() + ".txt";
    outputFileName = TESTDIR + "/TestStore-output-" + new Random().nextLong() + ".txt";

}

Example 15

Source File: TestGrunt.java From spork with Apache License 2.0

5 votes

@Test
public void testParsingAsInForeachWithOutBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast); "
            + "b = group a by foo; c = foreach b generate SUM(a.fast) as fast;\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 16

Source File: TestGrunt.java From spork with Apache License 2.0

5 votes

@Test
public void testBagConstantInForeachBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'input1'; "
            + "b = foreach a {generate {(1, '1', 0.4f),(2, '2', 0.45)};};\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 17

Source File: TestCombiner.java From spork with Apache License 2.0

5 votes

@Test
public void testSuccessiveUserFuncs1() throws Exception {
    String query = "a = load 'students.txt' as (c1,c2,c3,c4); " +
            "c = group a by c2; " +
            "f = foreach c generate COUNT(org.apache.pig.builtin.Distinct($1.$2)); " +
            "store f into 'out';";
    PigServer pigServer = new PigServer(cluster.getExecType(), properties);
    PigContext pc = pigServer.getPigContext();
    assertTrue((Util.buildMRPlan(Util.buildPp(pigServer, query), pc).getRoots().get(0).combinePlan
            .isEmpty()));
    pigServer.shutdown();
}

Example 18

Source File: TestCombiner.java From spork with Apache License 2.0

5 votes

@Test
public void testSuccessiveUserFuncs2() throws Exception {
    String dummyUDF = JiraPig1030.class.getName();
    String query = "a = load 'students.txt' as (c1,c2,c3,c4); " +
            "c = group a by c2; " +
            "f = foreach c generate COUNT(" + dummyUDF + "" +
            "(org.apache.pig.builtin.Distinct($1.$2)," + dummyUDF + "())); " +
            "store f into 'out';";
    PigServer pigServer = new PigServer(cluster.getExecType(), properties);
    PigContext pc = pigServer.getPigContext();
    assertTrue((Util.buildMRPlan(Util.buildPp(pigServer, query), pc).getRoots().get(0).combinePlan
            .isEmpty()));
    pigServer.shutdown();
}

Example 19

Source File: TestGrunt.java From spork with Apache License 2.0

5 votes

@Test
public void testExplainOut() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); explain -out /tmp -script "
            + basedir + "/testsubnested_run.pig;";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}

Example 20

Source File: BlackAndWhitelistFilter.java From spork with Apache License 2.0

4 votes

public BlackAndWhitelistFilter(PigServer pigServer) {
    this(pigServer.getPigContext());
}