Java Code Examples for org.apache.pig.tools.pigstats.PigStats#getOutputStats()

The following examples show how to use org.apache.pig.tools.pigstats.PigStats#getOutputStats() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestPigRunner.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDuplicateCounterName2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            if (outstats.getLocation().endsWith("tmp/output")) {
                assertEquals(2, outstats.getNumberRecords());
            } else {
                assertEquals(5, outstats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}
Example 2
Source File: TestPigRunner.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Test //PIG-1893
public void testEmptyFileCounter2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 < 0;");
    w1.println("store A into '" + OUTPUT_FILE + "';");
    w1.println("store B into 'output2';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            if (outstats.getLocation().endsWith("output2")) {
                assertEquals(0, outstats.getNumberRecords());
            } else {
                assertEquals(5, outstats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "output2");
    }
}
Example 3
Source File: TestPigRunner.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();

    try {
        String[] args = {"-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        if (execType.equals("tez")) {
            assertEquals(5, inputs.get(0).getNumberRecords());
            assertEquals(5, inputs.get(1).getNumberRecords());
        } else {
            for (InputStats instats : inputs) {
                // the multi-input counters are disabled
                assertEquals(-1, instats.getNumberRecords());
            }
        }

        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(1, outputs.size());
        OutputStats outstats = outputs.get(0);
        assertEquals(9, outstats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}
Example 4
Source File: TestPigRunner.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();

    try {
        String[] args = { "-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        if (execType.equals("tez")) {
            assertEquals(outputs.get(0).getNumberRecords(), 5);
            assertEquals(outputs.get(1).getNumberRecords(), 2);
        } else {
            for (OutputStats outstats : outputs) {
                // the multi-output counters are disabled
                assertEquals(-1, outstats.getNumberRecords());
            }
        }

        List<InputStats> inputs = stats.getInputStats();
        assertEquals(1, inputs.size());
        InputStats instats = inputs.get(0);
        assertEquals(5, instats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}