Java Code Examples for org.apache.pig.data.Tuple#set()

@Test
public void testBagFormat() throws Exception {
    DataBag bag = BagFactory.getInstance().newDefaultBag();

    Tuple tuple_1 = TupleFactory.getInstance().newTuple(1);
    tuple_1.set(0, 12);
    bag.add(tuple_1);

    Tuple tuple_2 = TupleFactory.getInstance().newTuple(1);
    DataBag innerBag = BagFactory.getInstance().newDefaultBag();
    innerBag.add(tuple_1);
    tuple_2.set(0, (innerBag));
    bag.add(tuple_2);

    System.out.println(BagFormat.format(bag));
    assertEquals("{(12),({(12)})}", BagFormat.format(bag));
}
 

@Test
public void testSerialize__nestedTuple() throws IOException {
    Tuple t = tf.newTuple(2);
    Tuple nestedTuple = tf.newTuple(2);
    nestedTuple.set(0, "sam");
    nestedTuple.set(1, "oes");
    t.set(0, nestedTuple);
    t.set(1, "basil");
    byte[] expectedOutput = "(sam,oes)\tbasil\n".getBytes();
    byte[] output = ps.serialize(t);
    Assert.assertArrayEquals(expectedOutput, output);
}
 

@Test
public void testTests() throws Exception {
    SearchEngineExtractor searchEngineExtractor = new SearchEngineExtractor();
    int testCount = 0;
    Tuple input=DefaultTupleFactory.getInstance().newTuple(1);
    for (String key : tests.keySet()) {
        String expected = tests.get(key);
        assertNotNull(expected);
        assertTrue(expected.length() > 0);
        input.set(0,key);
        assertEquals(expected, searchEngineExtractor.exec(input));
        testCount++;
    }
    assertEquals(tests.size(), testCount);
}
 

private Tuple toTuple(int[] digits) throws IOException, ExecException{
    Tuple t = mTupleFactory.newTuple(numInputs);
    for (int i=0; i<numInputs; i++){
        t.set(i, digits[i]);
    }
    return t;
}
 

@Test
public void testShouldThrowAnIOExceptionAsInputDoesNotContainDataBagOnly() {
    Tuple input = TupleFactory.getInstance().newTuple(2);
    try {
        input.set(0, "2012/05/05");
        input.set(1, bag());
        subtract.exec(input);
        fail("IOException Expected");
    } catch (IOException e) {
        assertEquals("Expecting input to be DataBag only but was 'String'", e.getMessage());
    }
}
 

/**
 * Parsing ISO date-less times works. 'T' prefix is required.
 * Date defaults to 1970-01-01
 * @throws Exception
 */
@Test
public void testParseDateTime_timeOnly_TimeZone() throws ExecException {

    Tuple t1 = TupleFactory.getInstance().newTuple(1);
    t1.set(0, "T08:11:33.020-0800");

    // Time zone is preserved.
    assertEquals(new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.forOffsetHours(-8)), ISOHelper.parseDateTime(t1));        
}
 

private Tuple getKeyTuple(Tuple t) throws ExecException
{
    Tuple key = TupleFactory.getInstance().newTuple(comparatorIndices.length);
    for (int i = 0; i < comparatorIndices.length; i++)
    {
        key.set(i, t.get(comparatorIndices[i]));
    }
    return key;
}
 

private static Tuple buildInputBag(DateTime ...dt) throws Exception
{
  Tuple input = TupleFactory.getInstance().newTuple(1);
  DataBag inputBag = BagFactory.getInstance().newDefaultBag();
  input.set(0,inputBag);

  for (DateTime time : dt)
  {
    inputBag.add(TupleFactory.getInstance().newTuple(Collections.singletonList(time.getMillis())));
  }

  return input;
}
 

public void testIEEEremainder() throws Exception {
    IEEEremainder rem = new IEEEremainder();
    Tuple tup = DefaultTupleFactory.getInstance().newTuple(2);
    try{
        tup.set(0, -0.5);
        tup.set(1,0.6);
    }catch (Exception e){}
    Double actual = rem.exec(tup);
    double expected = Math.IEEEremainder(-0.5,0.6);
    assertEquals(actual, expected, delta);
}
 

static public Tuple loadNestTuple(Tuple t, int[] input) throws ExecException {
    DataBag bag = BagFactory.getInstance().newDefaultBag();
    for(int i = 0; i < input.length; i++) {
        Tuple f = TupleFactory.getInstance().newTuple(1);
        f.set(0, input[i]);
        bag.add(f);
    }
    t.set(0, bag);
    return t;
}
 

@Test
public void testToWeek_Timezone() throws Exception {

    Tuple t1 = TupleFactory.getInstance().newTuple(1);
    t1.set(0, "2010-04-15T08:11:33.020-08:00");

    ISOToWeek func = new ISOToWeek();
    String truncated = func.exec(t1);

    assertEquals("2010-04-12T00:00:00.000-08:00", truncated);
}
 

@Test
public void testSerialize__map() throws IOException {
    Tuple t = tf.newTuple(1);
    Map<String, String> m = new TreeMap<String, String>();
    m.put("A", "B");
    m.put("C", "D");
    t.set(0,m);
    byte[] expectedOutput = "[A#B,C#D]\n".getBytes();
    byte[] output = ps.serialize(t);
    Assert.assertArrayEquals(expectedOutput, output);
}
 

public static void convertNullToUnknown(Tuple tuple) throws ExecException {
int idx = 0;
for(Object obj : tuple.getAll()) {
    if( (obj == null) ) {
	tuple.set(idx, unknown);
    }
    idx++;
}
   }
 

private Map<String, Object> getFindQuantilesResult(DataBag samples,
        int numReduceres) throws Exception {
    Tuple in = tFact.newTuple(2);

    in.set(0, new Integer(numReduceres));
    in.set(1, samples);
    
    FindQuantiles fq = new FindQuantiles();
    
    Map<String, Object> res = fq.exec(in);
    return res;
}
 

public static void copy(Tuple src, Tuple dest) throws ExecException
{
    int idx = 0;
    for (Object val : src.getAll())
    {
        dest.set(idx++, val);
    }
}
 

@Test
public void testToDay() throws Exception {

    Tuple t1 = TupleFactory.getInstance().newTuple(1);
    t1.set(0, "2010-04-15T08:11:33.020Z");

    ISOToDay func = new ISOToDay();
    String truncated = func.exec(t1);

    assertEquals("2010-04-15T00:00:00.000Z", truncated);
}
 

@Override
protected Tuple buildStruct(Tuple tuple) {
  for (Map.Entry<Integer, Object> e : partitionValues.entrySet()) {
    try {
      tuple.set(e.getKey(), e.getValue());
    } catch (ExecException ex) {
      throw new RuntimeException("Error setting value for key" + e.getKey(), ex);
    }
  }

  return tuple;
}
 

@Test
public void testMathFuncs() throws Exception {
    Random generator = new Random();
    generator.setSeed(System.currentTimeMillis());
    Double delta = 0.1;
    // We assume that UDFs are stored in org.apache.pig.builtin
    // Change this test case if we add more hierarchy later\
    // Also, we assume that we have a function with math function
    // associated with these UDF with a lowercase name
    String[] mathFuncs = {
            "SIN",
            "SINH",
            "ASIN",
            "COS",
            "COSH",
            "ACOS",
            "TAN",
            "TANH",
            "ATAN",
            "LOG",
            "LOG10",
            "SQRT",
            "CEIL",
            "EXP",
            "FLOOR",
            "CBRT"
    };
    String udfPackage = "org.apache.pig.builtin.";
    //String[] mathNonStdFuncs = {};
    EvalFunc<Double> evalFunc;
    Tuple tup;
    Double input, actual, expected;
    Method mathMethod;
    String msg;
    for (String func: mathFuncs) {
        evalFunc = (EvalFunc<Double>) Class.forName(udfPackage + func).newInstance();
        tup = TupleFactory.getInstance().newTuple(1);
        // double value between 0.0 and 1.0
        input = generator.nextDouble();
        tup.set(0, input);
        mathMethod = Math.class.getDeclaredMethod(func.toLowerCase(), double.class);
        actual = evalFunc.exec(tup);
        expected = (Double)mathMethod.invoke(null, input);
        msg = "[Testing " + func + " on input: " + input + " ( (actual) " + actual + " == " + expected + " (expected) )]";
        assertEquals(msg, actual, expected, delta);
    }
}
 

@Override
public Iterator<Tuple> call(Integer index, final 
		Iterator<Tuple> input) {
       Tuple inp = null;
       Tuple output = null;
       long sizeBag = 0L;

       List<Tuple> listOutput = new ArrayList<Tuple>();
       
       try {
       	while (input.hasNext()) {
			inp = input.next();
			output = TupleFactory.getInstance()
					.newTuple(inp.getAll().size() + 3);
			
			for (int i = 0; i < inp.getAll().size(); i++) {
				output.set(i + 3, inp.get(i));
			}
			
			if (poCounter.isRowNumber() || poCounter.isDenseRank()) {
				output.set(2, getLocalCounter());
				incrementSparkCounter();
				incrementLocalCounter();
			} else if (!poCounter.isDenseRank()) {
				int positionBag = inp.getAll().size()-1;
				if (inp.getType(positionBag) == DataType.BAG) {
	                sizeBag = ((org.apache.pig.data.DefaultAbstractBag)
	                		inp.get(positionBag)).size();
	            }
				
				output.set(2, getLocalCounter());
                
				addToSparkCounter(sizeBag);
                addToLocalCounter(sizeBag);
			}
			
			output.set(0, index);
			output.set(1, getSparkCounter());
			listOutput.add(output);
		}
       } catch(ExecException e) {
       	throw new RuntimeException(e);
       }
	
			
	return listOutput.iterator();
}
 

@Override
public Tuple exec(Tuple input) throws IOException {

	// check
	int dummy = 0;
	
	// Initialize Evaluator if null:
	if (this.evaluator == null) {
		try {
			System.out.println("Initializing: "+(dummy++)+" time");
			Schema inputSchema = getInputSchema();
			this.initialize(inputSchema);			// something to check
		} catch (Throwable t) {
			throw new RuntimeException("Backend: Unable to initialize PMML file: ",t);
		}
	}

	// Initialize Output as Input
	Tuple outputTuple = tf.newTuple(this.predictedFields.size() + this.outputFields.size());

	/* ************************
	// BLOCK: Prepare Data
	************************* */
	
	for(FieldName inputField : this.activeFields){

		// Get Object
		Object origBodyCell = (Object) input.get(aliasMap.get(inputField.getValue().toLowerCase()));
		
		Object bodyCell;
		if (origBodyCell instanceof Long) {
			bodyCell = ((Long) origBodyCell).doubleValue();
		} else {
			bodyCell = origBodyCell;
		}

		// Prepare Object for Scoring
		this.preparedRow.put(inputField, EvaluatorUtil.prepare(this.evaluator, inputField, bodyCell));

		// Prepare Object for Scoring
		// CC: Removed this b/c I think the "Long" check above resolves any issues.
		/*
		try {
			this.preparedRow.put(inputField, EvaluatorUtil.prepare(this.evaluator, inputField, bodyCell));
		} catch (Throwable t) {
        	System.err.println("Unable to prepare record, Trouble Parsing: " + inputField.toString() + " (value="+ bodyCell+")");
        	System.err.println(t);
            throw new RuntimeException(t);
		}
		*/

	}

	// Score Data
	Map<FieldName, ?> result = evaluator.evaluate(this.preparedRow);

	// Append Predicted Fields
	int i = 0;
	for(FieldName predictedField : this.predictedFields){
		outputTuple.set(i++,EvaluatorUtil.decode(result.get(predictedField)));
	}

	for(FieldName outputField : this.outputFields){
		outputTuple.set(i++,EvaluatorUtil.decode(result.get(outputField)));
	}

	// Return Tuple:
	return outputTuple;

}