Java Code Examples for org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit#getNumPaths()
The following examples show how to use
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit#getNumPaths() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HadoopExecutableManager.java From spork with Apache License 2.0 | 5 votes |
|
private void writeDebugHeader() {
processError("===== Task Information Header =====" );
processError("\nCommand: " + command);
processError("\nStart time: " + new Date(System.currentTimeMillis()));
if (job.getBoolean(MRConfiguration.TASK_IS_MAP, false)) {
MapContext context = (MapContext)PigMapReduce.sJobContext;
PigSplit pigSplit = (PigSplit)context.getInputSplit();
int numPaths = pigSplit.getNumPaths();
processError("\nPigSplit contains " + numPaths + " wrappedSplits.");
StringBuilder sb = new StringBuilder();
for(int i = 0; i < numPaths; i++) {
InputSplit wrappedSplit = pigSplit.getWrappedSplit(i);
if (wrappedSplit instanceof FileSplit) {
FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
sb.append("\nInput-split: file=");
sb.append(mapInputFileSplit.getPath());
sb.append(" start-offset=");
sb.append(Long.toString(mapInputFileSplit.getStart()));
sb.append(" length=");
sb.append(Long.toString(mapInputFileSplit.getLength()));
processError(sb.toString());
sb.setLength(0);
}
}
}
processError("\n===== * * * =====\n");
}
Example 2
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 5 votes |
|
@Test
public void test1() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l1", "l4", "l5"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(result.size(), 2);
int index = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
if (index == 0) {
Assert.assertEquals(2, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(500, pigSplit.getLength(0));
Assert.assertEquals(400, pigSplit.getLength(1));
}
else {
Assert.assertEquals(1, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l4", "l5"
});
Assert.assertEquals(400, pigSplit.getLength(0));
}
index++;
}
}
Example 3
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 5 votes |
|
@Test
public void test3() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l4", "l5"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(1, result.size());
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
Assert.assertEquals(3, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3", "l4", "l5"
});
Assert.assertEquals(500, pigSplit.getLength(0));
Assert.assertEquals(200, pigSplit.getLength(1));
Assert.assertEquals(100, pigSplit.getLength(2));
}
}
Example 4
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 5 votes |
|
@Test
public void test8() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l1", "l4", "l5"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(result.size(), 1);
int index = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
Assert.assertEquals(3, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3", "l4", "l5"
});
Assert.assertEquals(200, pigSplit.getLength(0));
Assert.assertEquals(100, pigSplit.getLength(1));
Assert.assertEquals(100, pigSplit.getLength(2));
index++;
}
}
Example 5
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 5 votes |
|
@Test
public void test9() throws IOException, InterruptedException {
// verify locations in order
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l3", "l4", "l5"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l5", "l6", "l1"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(result.size(), 1);
int index = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
Assert.assertEquals(3, len);
// only 5 locations are in list: refer to PIG-1648 for more details
checkLocationOrdering(pigSplit.getLocations(), new String[] {
"l5", "l1", "l6", "l3", "l4"
});
Assert.assertEquals(400, pigSplit.getLength(0));
Assert.assertEquals(200, pigSplit.getLength(1));
Assert.assertEquals(100, pigSplit.getLength(2));
index++;
}
}
Example 6
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void test2() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(600, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(700, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(800, new String[] {
"l1", "l4", "l5"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(result.size(), 3);
int index = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
if (index == 0) {
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l4", "l5"
});
Assert.assertEquals(1, len);
Assert.assertEquals(800, pigSplit.getLength(0));
}
else if (index == 1) {
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(1, len);
Assert.assertEquals(700, pigSplit.getLength(0));
}
else {
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(1, len);
Assert.assertEquals(600, pigSplit.getLength(0));
}
index++;
}
}
Example 7
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void test4() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l4", "l5"
}));
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l4", "l5"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(2, result.size());
int idx = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
if (idx == 0) {
Assert.assertEquals(2, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l4", "l5"
});
Assert.assertEquals(500, pigSplit.getLength(0));
Assert.assertEquals(100, pigSplit.getLength(1));
}
else {
Assert.assertEquals(4, len);
Assert.assertEquals(500, pigSplit.getLength(0));
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(200, pigSplit.getLength(1));
Assert.assertEquals(200, pigSplit.getLength(2));
Assert.assertEquals(100, pigSplit.getLength(3));
}
idx++;
}
}
Example 8
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void test5() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l1", "l4", "l5"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, false, conf);
Assert.assertEquals(3, result.size());
int index = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
if (index == 0) {
Assert.assertEquals(1, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(500, pigSplit.getLength(0));
}
else if (index == 1) {
Assert.assertEquals(1, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(400, pigSplit.getLength(0));
}
else {
Assert.assertEquals(1, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l4", "l5"
});
Assert.assertEquals(400, pigSplit.getLength(0));
}
index++;
}
}
Example 9
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void test6() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(600, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(300, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l2", "l3"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(3, result.size());
int idx = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
if (idx == 0) {
Assert.assertEquals(2, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(600, pigSplit.getLength(0));
Assert.assertEquals(400, pigSplit.getLength(1));
}
else if (idx == 1) {
Assert.assertEquals(3, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(500, pigSplit.getLength(0));
Assert.assertEquals(300, pigSplit.getLength(1));
Assert.assertEquals(200, pigSplit.getLength(2));
}
else {
Assert.assertEquals(1, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(100, pigSplit.getLength(0));
}
idx++;
}
}
Example 10
| Source File: TestSplitCombine.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void test7() throws IOException, InterruptedException {
ArrayList<InputSplit> rawSplits = new ArrayList<InputSplit>();
rawSplits.add(new DummyInputSplit(100, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(200, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(300, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(400, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(500, new String[] {
"l1", "l2", "l3"
}));
rawSplits.add(new DummyInputSplit(600, new String[] {
"l1", "l2", "l3"
}));
List<InputSplit> result = pigInputFormat.getPigSplits(rawSplits, 0, ok,
null, true, conf);
Assert.assertEquals(3, result.size());
int idx = 0;
for (InputSplit split : result) {
PigSplit pigSplit = (PigSplit) split;
int len = pigSplit.getNumPaths();
if (idx == 0) {
Assert.assertEquals(2, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(600, pigSplit.getLength(0));
Assert.assertEquals(400, pigSplit.getLength(1));
}
else if (idx == 1) {
Assert.assertEquals(3, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(500, pigSplit.getLength(0));
Assert.assertEquals(300, pigSplit.getLength(1));
Assert.assertEquals(200, pigSplit.getLength(2));
}
else {
Assert.assertEquals(1, len);
checkLocations(pigSplit.getLocations(), new String[] {
"l1", "l2", "l3"
});
Assert.assertEquals(100, pigSplit.getLength(0));
}
idx++;
}
}
