Java Code Examples for org.apache.pig.data.DataType#toLong()

The following examples show how to use org.apache.pig.data.DataType#toLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestAlgebraicEval.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSimpleCount() throws Exception {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        System.err.println("Testing testSimpleCount with null flag:" + nullFlags[i]);

        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        generateInput(ps, nullFlags[i]);
        String query = "myid =  foreach (group (load '"
            + Util.generateURI(tmpFile.toString(), pig.getPigContext())
            + "') all) generate COUNT($1);";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple> it = pig.openIterator("myid");
        tmpFile.delete();
        Tuple t = it.next();
        Long count = DataType.toLong(t.get(0));
        assertEquals(this.getClass().getName() + "with nullFlags set to: "
                + nullFlags[i], count.longValue(), LOOP_COUNT);
    }
}
Example 2
Source File: TestAlgebraicEval.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGroupCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);

        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        generateInput(ps, nullFlags[i]);
        String query = "myid = foreach (group (load '"
            + Util.generateURI(tmpFile.toString(), pig.getPigContext())
            + "') all) generate group, COUNT($1) ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple>  it = pig.openIterator("myid");
        tmpFile.delete();
        Tuple t = it.next();
        Long count = DataType.toLong(t.get(1));
        assertEquals(this.getClass().getName() + "with nullFlags set to: "
                + nullFlags[i], count.longValue(), LOOP_COUNT);
    }
}
Example 3
Source File: TestAlgebraicEval.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGroupReorderCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        generateInput(ps, nullFlags[i]);
        String query = "myid = foreach (group (load '"
            + Util.generateURI(tmpFile.toString(), pig.getPigContext())
            + "') all) generate COUNT($1), group ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple>  it = pig.openIterator("myid");
        tmpFile.delete();
        Tuple t = it.next();
        Long count = DataType.toLong(t.get(0));
        assertEquals(this.getClass().getName() + "with nullFlags set to: "
                + nullFlags[i], count.longValue(), LOOP_COUNT);
    }
}
Example 4
Source File: TestBuiltin.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCOUNTIntermed() throws Exception {
    Integer input[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };

    DataBag intermediateInputBag = bagFactory.newDefaultBag();
    // call initial and then Intermed
    for (Integer i : input) {
        Tuple t = tupleFactory.newTuple(i);
        DataBag b = bagFactory.newDefaultBag();
        b.add(t);
        Tuple initialInput = tupleFactory.newTuple(b);
        EvalFunc<?> initial = new COUNT.Initial();
        intermediateInputBag.add((Tuple)initial.exec(initialInput));
    }

    EvalFunc<Tuple> countIntermed = new COUNT.Intermediate();
    Tuple intermediateInput = tupleFactory.newTuple(intermediateInputBag);
    Tuple output = countIntermed.exec(intermediateInput);

    Long f1 = DataType.toLong(output.get(0));
    assertEquals("Expected count to be 10", 10, f1.longValue());
}
Example 5
Source File: TestBuiltin.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCOUNT_STARIntermed() throws Exception {
    Integer input[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };

    DataBag intermediateInputBag = bagFactory.newDefaultBag();
    // call initial and then Intermed
    for (Integer i : input) {
        Tuple t = tupleFactory.newTuple(i);
        DataBag b = bagFactory.newDefaultBag();
        b.add(t);
        Tuple initialInput = tupleFactory.newTuple(b);
        EvalFunc<?> initial = new COUNT_STAR.Initial();
        intermediateInputBag.add((Tuple)initial.exec(initialInput));
    }

    EvalFunc<Tuple> countIntermed = new COUNT_STAR.Intermediate();
    Tuple intermediateInput = tupleFactory.newTuple(intermediateInputBag);
    Tuple output = countIntermed.exec(intermediateInput);

    Long f1 = DataType.toLong(output.get(0));
    assertEquals("Expected count to be 10", 10, f1.longValue());
}
Example 6
Source File: TestAlgebraicEvalLocal.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSimpleCount() throws Exception {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        System.err.println("Testing testSimpleCount with null flag:" + nullFlags[i]);

        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        generateInput(ps, nullFlags[i]);
        String query = "myid =  foreach (group (load '"
                + Util.generateURI(tmpFile.toString(), pig.getPigContext())
                + "') all) generate COUNT($1);";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple> it = pig.openIterator("myid");
        tmpFile.delete();
        Tuple t = it.next();
        Long count = DataType.toLong(t.get(0));
        assertEquals(this.getClass().getName() + "with nullFlags set to: "
                + nullFlags[i], count.longValue(), LOOP_COUNT);
    }
}
Example 7
Source File: TestAlgebraicEvalLocal.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGroupCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);

        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        generateInput(ps, nullFlags[i]);
        String query = "myid = foreach (group (load '"
                + Util.generateURI(tmpFile.toString(), pig.getPigContext())
                + "') all) generate group, COUNT($1) ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple> it = pig.openIterator("myid");
        tmpFile.delete();
        Tuple t = it.next();
        Long count = DataType.toLong(t.get(1));
        assertEquals(this.getClass().getName() + "with nullFlags set to: "
                + nullFlags[i], count.longValue(), LOOP_COUNT);
    }
}
Example 8
Source File: TestAlgebraicEvalLocal.java    From spork with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGroupReorderCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        generateInput(ps, nullFlags[i]);
        String query = "myid = foreach (group (load '"
                + Util.generateURI(tmpFile.toString(), pig.getPigContext())
                + "') all) generate COUNT($1), group ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple> it = pig.openIterator("myid");
        tmpFile.delete();
        Tuple t = it.next();
        Long count = DataType.toLong(t.get(0));
        assertEquals(this.getClass().getName() + "with nullFlags set to: "
                + nullFlags[i], count.longValue(), LOOP_COUNT);
    }
}
Example 9
Source File: UnixToISO.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Override
public String exec(Tuple input) throws IOException
{
    if (input == null || input.size() < 1 || input.get(0) == null) {
        return null;
    }
    
    DateTime result = new DateTime(DataType.toLong(input.get(0)));

    return result.toString();
}
Example 10
Source File: ToDate.java    From spork with Apache License 2.0 4 votes vote down vote up 
public DateTime exec(Tuple input) throws IOException {
    if (input == null || input.size() < 1 || input.get(0) == null) {
        return null;
    }
    return new DateTime(DataType.toLong(input.get(0)));
}
Example 11
Source File: TestAlgebraicEval.java    From spork with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGroupUniqueColumnCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        long groupsize = 0;
        if(nullFlags[i] == false) {
            // generate data without nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) groupsize++;
                ps.println(j%10 + ":" + j);
            }
        } else {
            // generate data with nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) groupsize++;
                if(j % 20 == 0) {
                    // for half the groups
                    // emit nulls
                    ps.println(j%10 + ":");
                    groupsize--;
                } else {
                    ps.println(j%10 + ":" + j);
                }
            }
        }
        ps.close();
        String query = "myid = foreach (group (load '"
            + Util.generateURI(tmpFile.toString(), pig.getPigContext())
            + "' using " + PigStorage.class.getName()
            + "(':')) by $0) generate group, COUNT($1.$1) ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple>  it = pig.openIterator("myid");
        tmpFile.delete();
        System.err.println("Output from testGroupUniqueColumnCount");
        while(it.hasNext()) {
            Tuple t = it.next();
            System.err.println(t);
            String a = t.get(0).toString();
            Double group = Double.valueOf(a.toString());
            if(group == 0.0) {
                Long count = DataType.toLong(t.get(1));
                assertEquals(this.getClass().getName() + "with nullFlags set to: "
                        + nullFlags[i], groupsize, count.longValue());
            }
        }
    }
}
Example 12
Source File: TestAlgebraicEval.java    From spork with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGroupDuplicateColumnCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        long groupsize = 0;
        long nonNullCnt = 0;
        if(nullFlags[i] == false) {
            // generate data without nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) {groupsize++; nonNullCnt++;}
                ps.println(j%10 + ":" + j);
            }
        } else {
            // generate data with nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) {groupsize++; nonNullCnt++;}
                if(j % 20 == 0) {
                    // for half the groups
                    // emit nulls
                    nonNullCnt--;
                    ps.println(j%10 + ":");
                } else {
                    ps.println(j%10 + ":" + j);
                }
            }
        }
        ps.close();
        String query = "myid = foreach (group (load '"
            + Util.generateURI(tmpFile.toString(), pig.getPigContext())
            + "' using " + PigStorage.class.getName()
            + "(':')) by $0) generate group, COUNT($1.$1), COUNT($1.$0) ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple>  it = pig.openIterator("myid");
        tmpFile.delete();
        System.err.println("Output from testGroupDuplicateColumnCount");
        while(it.hasNext()) {
            Tuple t = it.next();
            System.err.println(t);
            String a = t.get(0).toString();
            Double group = Double.valueOf(a.toString());
            if(group == 0.0) {
                Long count = DataType.toLong(t.get(2));
                assertEquals(this.getClass().getName() + "with nullFlags set to: "
                        + nullFlags[i],groupsize, count.longValue());
                count = DataType.toLong(t.get(1));
                assertEquals(this.getClass().getName() + "with nullFlags set to: "
                        + nullFlags[i],nonNullCnt, count.longValue());
            }
        }
    }
}
Example 13
Source File: TestAlgebraicEvalLocal.java    From spork with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGroupUniqueColumnCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        long groupsize = 0;
        if(nullFlags[i] == false) {
            // generate data without nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) groupsize++;
                ps.println(j%10 + ":" + j);
            }
        } else {
            // generate data with nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) groupsize++;
                if(j % 20 == 0) {
                    // for half the groups
                    // emit nulls
                    ps.println(j%10 + ":");
                    groupsize --;
                } else {
                    ps.println(j%10 + ":" + j);
                }
            }
        }
        ps.close();
        String query = "myid = foreach (group (load '"
                + Util.generateURI(tmpFile.toString(), pig.getPigContext())
                + "' using " + PigStorage.class.getName()
                + "(':')) by $0) generate group, COUNT($1.$1) ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple> it = pig.openIterator("myid");
        tmpFile.delete();
        System.err.println("Output from testGroupUniqueColumnCount");
        while(it.hasNext()) {
            Tuple t = it.next();
            System.err.println(t);
            String a = t.get(0).toString();
            Double group = Double.valueOf(a.toString());
            if(group == 0.0) {
                Long count = DataType.toLong(t.get(1));
                assertEquals(this.getClass().getName() + "with nullFlags set to: "
                        + nullFlags[i], groupsize, count.longValue());
            }
        }
    }
}
Example 14
Source File: TestAlgebraicEvalLocal.java    From spork with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGroupDuplicateColumnCount() throws Throwable {
    File tmpFile = File.createTempFile("test", "txt");
    for (int i = 0; i < nullFlags.length; i++) {
        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
        long groupsize = 0;
        long nonNullCnt = 0;
        if(nullFlags[i] == false) {
            // generate data without nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) {groupsize++; nonNullCnt++;}
                ps.println(j%10 + ":" + j);
            }
        } else {
            // generate data with nulls
            for(int j = 0; j < LOOP_COUNT; j++) {
                if(j%10 == 0) {groupsize++; nonNullCnt++;}
                if(j % 20 == 0) {
                    // for half the groups
                    // emit nulls
                    ps.println(j%10 + ":");
                    nonNullCnt--;
                } else {
                    ps.println(j%10 + ":" + j);
                }
            }
        }
        ps.close();
        String query = "myid = foreach (group (load '"
                + Util.generateURI(tmpFile.toString(), pig.getPigContext())
                + "' using "
                + PigStorage.class.getName()
                + "(':')) by $0) generate group, COUNT($1.$1), COUNT($1.$0) ;";
        System.out.println(query);
        pig.registerQuery(query);
        Iterator<Tuple> it = pig.openIterator("myid");
        tmpFile.delete();
        System.err.println("Output from testGroupDuplicateColumnCount");
        while(it.hasNext()) {
            Tuple t = it.next();
            System.err.println(t);
            String a = t.get(0).toString();
            Double group = Double.valueOf(a.toString());
            if(group == 0.0) {
                Long count = DataType.toLong(t.get(2));
                assertEquals(this.getClass().getName() + "with nullFlags set to: "
                        + nullFlags[i], groupsize, count.longValue());
                count = DataType.toLong(t.get(1));
                assertEquals(this.getClass().getName() + "with nullFlags set to: "
                        + nullFlags[i], nonNullCnt, count.longValue());
            }
        }
    }
}
Example 15
Source File: Coalesce.java    From datafu with Apache License 2.0 4 votes vote down vote up 
@Override
public Object exec(Tuple input) throws IOException
{    
  if (input == null || input.size() == 0)
  {
    return null;
  }
  
  Byte type = (Byte)getInstanceProperties().get("type");
              
  for (Object o : input)
  {
    if (o != null)
    {
      if (strict)
      {
        return o;
      }
      else
      {
        try
        {
          switch (type)
          {
          case DataType.INTEGER:
            return DataType.toInteger(o);
          case DataType.LONG:
            return DataType.toLong(o);
          case DataType.DOUBLE:
            return DataType.toDouble(o); 
          case DataType.FLOAT:
            return DataType.toFloat(o);      
          default:
            return o;
          }
        }
        catch (Exception e)
        {
          DataFuException dfe = new DataFuException(e.getMessage(),e);
          dfe.setData(o);
          dfe.setFieldAliases(getFieldAliases());
          throw dfe;
        }
      }
    }
  }
  
  return null;
}