org.apache.pig.data.DataByteArray Java Examples

public static PyObject pigToPython(Object object) {
    if (object instanceof Tuple) {
        return pigTupleToPyTuple((Tuple) object);
    } else if (object instanceof DataBag) {
        PyList list = new PyList();
        for (Tuple bagTuple : (DataBag) object) {
            list.add(pigTupleToPyTuple(bagTuple));
        }
        return list;
    } else if (object instanceof Map<?, ?>) {
        PyDictionary newMap = new PyDictionary();
        for (Map.Entry<?, ?> entry : ((Map<?, ?>) object).entrySet()) {
            newMap.put(entry.getKey(), pigToPython(entry.getValue()));
        }
        return newMap;
    } else if (object instanceof DataByteArray) {
        return Py.java2py(((DataByteArray) object).get());
    } else {
        return Py.java2py(object);
    }
}
 

/**
 *
 * @param caster LoadCaster to be used to convert the bytes into a field.
 * @param bytes
 * @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
 * @param dataType type from DataType
 * @return converted object.
 * @throws IOException
 */
public static Object convertToType(LoadCaster caster, byte[] bytes,
        ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
    switch (dataType) {
    case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
    case (DataType.BYTEARRAY): return new DataByteArray(bytes);
    case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
    case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
    case (DataType.FLOAT): return caster.bytesToFloat(bytes);
    case (DataType.INTEGER): return caster.bytesToInteger(bytes);
    case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
    case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
    case (DataType.LONG): return caster.bytesToLong(bytes);
    case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
    case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
    case (DataType.MAP): return caster.bytesToMap(bytes, fieldSchema);
    case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
    default: throw new IOException("Unknown type " + dataType);
    }
}
 

/**
 * load from hbase test without hbase:// prefix
 *
 * @throws IOException
 */
@Test
public void testBackwardsCompatibility() throws IOException {
    prepareTable(TESTTABLE_1, true, DataFormat.UTF8PlainText);
    pig.registerQuery("a = load '" + TESTTABLE_1 + "' using "
            + "org.apache.pig.backend.hadoop.hbase.HBaseStorage('"
            + TESTCOLUMN_A + " " + TESTCOLUMN_B + " " + TESTCOLUMN_C
            + "') as (col_a, col_b, col_c);");
    Iterator<Tuple> it = pig.openIterator("a");
    int count = 0;
    LOG.info("BackwardsCompatibility Starting");
    while (it.hasNext()) {
        Tuple t = it.next();
        String col_a = ((DataByteArray) t.get(0)).toString();
        String col_b = ((DataByteArray) t.get(1)).toString();
        String col_c = ((DataByteArray) t.get(2)).toString();

        Assert.assertEquals(count, Integer.parseInt(col_a));
        Assert.assertEquals(count + 0.0, Double.parseDouble(col_b), 1e-6);
        Assert.assertEquals("Text_" + count, col_c);
        count++;
    }
    Assert.assertEquals(TEST_ROW_COUNT, count);
    LOG.info("BackwardsCompatibility done");
}
 

@Test
public void testByteArrayCast15() throws IOException, ExecException {
    // Passing (bytearray)
    // Possible matches: (bytearray), (int)
    // Chooses (bytearray) because that is an exact match
    pigServer.registerQuery("A = LOAD '" + inputFile + "' as (x, y);");
    pigServer.registerQuery("B = FOREACH A generate " + UDF3.class.getName() + "(y);");
    Iterator<Tuple> iter = pigServer.openIterator("B");
    assertTrue("No Output received", iter.hasNext());
    int cnt = 0;
    while (iter.hasNext()) {
        Tuple t = iter.next();
        assertTrue(((Tuple)t.get(0)).get(0) instanceof DataByteArray);
        byte[] expected = Integer.toString(cnt + 1).getBytes();
        byte[] actual = ((DataByteArray)((Tuple)t.get(0)).get(0)).get();
        assertEquals(expected.length, actual.length);
        for (int i = 0; i < expected.length; i++) {
            assertEquals(expected[i], actual[i]);
        }
        ++cnt;
    }
    assertEquals(LOOP_SIZE, cnt);
}
 

public static boolean mapEquals(Map<String, Object> expectedMap, Map<String, Object> convertedMap) {
    if(expectedMap == null) {
        if(convertedMap != null) {
            return false;
        }
    } else {
        if (convertedMap == null) {
            return false;
        }
    }
    
    if(expectedMap.size() != convertedMap.size()) {
        return false;
    }
    
    for(String key: expectedMap.keySet()) {
        Object v = convertedMap.get(key);
        String convertedValue = new String(((DataByteArray)v).get());
        if(!expectedMap.get(key).toString().equals(convertedValue)) {
            return false;
        }
    }
    return true;
}
 

public void testLoadMyRegExFromPigServer() throws Exception {
  ArrayList<DataByteArray[]> expected = TestHelper.getExpected(data, pattern);
  String filename = TestHelper.createTempFile(data, "");
  PigServer pig = new PigServer(LOCAL);
  filename = filename.replace("\\", "\\\\");
  patternString = patternString.replace("\\", "\\\\");
  String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.MyRegExLoader('" + patternString + "');";
  pig.registerQuery(query);
  Iterator<?> it = pig.openIterator("A");
  int tupleCount = 0;
  while (it.hasNext()) {
    Tuple tuple = (Tuple) it.next();
    if (tuple == null)
      break;
    else {
      TestHelper.examineTuple(expected, tuple, tupleCount);
      tupleCount++;
    }
  }
  assertEquals(data.size(), tupleCount);
}
 

public static ArrayList<DataByteArray[]> getExpected(ArrayList<String[]> data, Pattern pattern) {
    ArrayList<DataByteArray[]> expected = new ArrayList<DataByteArray[]>();
    for (int i = 0; i < data.size(); i++) {
        String string = data.get(i)[0];
        Matcher matcher = pattern.matcher(string);
        matcher.groupCount();
        matcher.find();
        DataByteArray[] toAdd = new DataByteArray[] { 
            new DataByteArray(matcher.group(1)), 
            new DataByteArray(matcher.group(2)), 
            new DataByteArray(matcher.group(3)) };
        expected.add(toAdd);
    }

    return expected;
}
 

@Test
public void testOnlyLastMatch() throws Exception {       
    PigServer pigServer = new PigServer(LOCAL);
    
    String filename = TestHelper.createTempFile(data, "");

	ArrayList<String[]> dataE = new ArrayList<String[]>();
    dataE.add(new String[] { "3,three;iii" });
   	ArrayList<DataByteArray[]> expected = TestHelper.getExpected(dataE, pattern2);
    
    pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(filename) + 
            "' USING " + DummyRegExLoader2.class.getName() + "() AS (key, val);");
    Iterator<?> it = pigServer.openIterator("A");
    int tupleCount = 0;
    while (it.hasNext()) {
        Tuple tuple = (Tuple) it.next();
        if (tuple == null)
          break;
        else {
          TestHelper.examineTuple(expected, tuple, tupleCount);
          tupleCount++;
        }
      }
    assertEquals(1, tupleCount);
}
 

@Test
public void testStringTypeInTuple() throws IOException {
    Query query = new Query();
    query.value = "";
    Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.BYTE),
                                  makeFieldSchema("b", DataType.BYTEARRAY),
                                  makeFieldSchema("c", DataType.CHARARRAY));
    Tuple fakeTuple = makeTuple(Byte.valueOf("1"), new DataByteArray("foo".getBytes()), "bar");

    sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple));
    runWithoutOutput(() -> sty.execute(query));
    Assert.assertFalse(query.failed());
    List<TypedObject> columnOne = query.getResult().getColumn("a").getValues();
    List<TypedObject> columnTwo = query.getResult().getColumn("b").getValues();
    List<TypedObject> columnThree = query.getResult().getColumn("c").getValues();
    Assert.assertNotNull(columnOne);
    Assert.assertEquals(columnOne.size(), 1);
    Assert.assertEquals(columnOne.get(0).data, "1");
    Assert.assertNotNull(columnTwo);
    Assert.assertEquals(columnTwo.size(), 1);
    Assert.assertEquals(columnTwo.get(0).data, "foo");
    Assert.assertNotNull(columnThree);
    Assert.assertEquals(columnThree.size(), 1);
    Assert.assertEquals(columnThree.get(0).data, "bar");
}
 

@Test
public void testDataByteArrayLt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("a"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    GTOrEqualToExpr g = GenPhyOp.compGTOrEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);
}
 

@Override
public Map<String, Object> exec(Tuple input) throws IOException {

    TupleFactory tupleFactory = TupleFactory.getInstance();
    ArrayList<Object> objList = new ArrayList<Object>();
    objList.add(new Integer(1));
    objList.add(new Double(1.0));
    objList.add(new Float(1.0));
    objList.add(new String("World!"));
    Tuple tuple = tupleFactory.newTuple(objList);

    BagFactory bagFactory = BagFactory.getInstance();
    DataBag bag = bagFactory.newDefaultBag();
    bag.add(tuple);

    Map<String, Object> mapInMap = new HashMap<String, Object>();
    mapInMap.put("int", new Integer(10));
    mapInMap.put("float", new Float(10.0));

    Map<String, Object> myMap = new HashMap<String, Object>();
    myMap.put("string", new String("Hello"));
    myMap.put("int", new Integer(1));
    myMap.put("long", new Long(1));
    myMap.put("float", new Float(1.0));
    myMap.put("double", new Double(1.0));
    myMap.put("dba", new DataByteArray(new String("bytes").getBytes()));
    myMap.put("map", mapInMap);
    myMap.put("tuple", tuple);
    myMap.put("bag", bag);
    return myMap;
}
 

@SuppressWarnings("unchecked")
private byte[] objToBytes(Object o, byte type) throws IOException {
    LoadStoreCaster caster = (LoadStoreCaster) caster_;
    if (o == null) return null;
    switch (type) {
    case DataType.BYTEARRAY: return ((DataByteArray) o).get();
    case DataType.BAG: return caster.toBytes((DataBag) o);
    case DataType.CHARARRAY: return caster.toBytes((String) o);
    case DataType.DOUBLE: return caster.toBytes((Double) o);
    case DataType.FLOAT: return caster.toBytes((Float) o);
    case DataType.INTEGER: return caster.toBytes((Integer) o);
    case DataType.LONG: return caster.toBytes((Long) o);
    case DataType.BIGINTEGER: return caster.toBytes((BigInteger) o);
    case DataType.BIGDECIMAL: return caster.toBytes((BigDecimal) o);
    case DataType.BOOLEAN: return caster.toBytes((Boolean) o);
    case DataType.DATETIME: return caster.toBytes((DateTime) o);

    // The type conversion here is unchecked.
    // Relying on DataType.findType to do the right thing.
    case DataType.MAP: return caster.toBytes((Map<String, Object>) o);

    case DataType.NULL: return null;
    case DataType.TUPLE: return caster.toBytes((Tuple) o);
    case DataType.ERROR: throw new IOException("Unable to determine type of " + o.getClass());
    default: throw new IOException("Unable to find a converter for tuple field " + o);
    }
}
 

@Test
public void testExpressionTypeChecking2() throws Throwable {
    LogicalExpressionPlan plan = new LogicalExpressionPlan();
    ConstantExpression constant1 = new ConstantExpression(plan, 10);
    ConstantExpression constant2 =  new ConstantExpression(plan,  new DataByteArray());
    ConstantExpression constant3 =  new ConstantExpression(plan,  123L);
    ConstantExpression constant4 =  new ConstantExpression(plan,  true);

    SubtractExpression sub1 = new SubtractExpression(plan, constant1, constant2);
    GreaterThanExpression gt1 = new GreaterThanExpression(plan, sub1, constant3);
    AndExpression and1 = new AndExpression(plan, gt1, constant4);
    NotExpression not1 = new NotExpression(plan, and1);

    CompilationMessageCollector collector = new CompilationMessageCollector();
    TypeCheckingExpVisitor expTypeChecker = new TypeCheckingExpVisitor(plan, collector, null);
    expTypeChecker.visit();


    printMessageCollector(collector);
    //printTypeGraph(plan);

    if (collector.hasError()) {
        throw new Exception("Error not expected during type checking");
    }


    // Induction check
    assertEquals(DataType.INTEGER, sub1.getType());
    assertEquals(DataType.BOOLEAN, gt1.getType());
    assertEquals(DataType.BOOLEAN, and1.getType());
    assertEquals(DataType.BOOLEAN, not1.getType());

    // Cast insertion check
    assertEquals(DataType.INTEGER, sub1.getRhs().getType());
    assertEquals(DataType.LONG, gt1.getLhs().getType());

}
 

/**
 *     * Test Load from hbase with map parameters and column prefix with a
 *     static column
 *
 */
@Test
public void testLoadWithFixedAndPrefixedCols() throws IOException {
    prepareTable(TESTTABLE_1, true, DataFormat.UTF8PlainText);

    //NOTE: I think there is some strangeness in how HBase handles column
    // filters. I was only able to reproduce a bug related to missing column
    // values in the response when I used 'sc' as a column name, instead of
    // 'col_a' as I use below.
    pig.registerQuery("a = load 'hbase://"
            + TESTTABLE_1
            + "' using "
            + "org.apache.pig.backend.hadoop.hbase.HBaseStorage('"
            + "pig:sc pig:prefixed_col_*"
            + "','-loadKey') as (rowKey:chararray, sc:chararray, pig_cf_map:map[]);");
    Iterator<Tuple> it = pig.openIterator("a");
    int count = 0;
    LOG.info("LoadFromHBase Starting");
    while (it.hasNext()) {
        Tuple t = it.next();
        LOG.info("LoadFromHBase " + t);
        String rowKey = (String) t.get(0);
        String col_a = t.get(1) != null ? t.get(1).toString() : null;
        Map pig_cf_map = (Map) t.get(2);
        Assert.assertEquals(3, t.size());

        Assert.assertEquals("00".substring((count + "").length()) + count,
                rowKey);
        Assert.assertEquals("PrefixedText_" + count,
                ((DataByteArray) pig_cf_map.get("prefixed_col_d")).toString());
        Assert.assertEquals(1, pig_cf_map.size());
        Assert.assertEquals(Integer.toString(count), col_a);

        count++;
    }
    Assert.assertEquals(TEST_ROW_COUNT, count);
    LOG.info("LoadFromHBase done");
}
 

@Test
public void testDataByteArrayLt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("a"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    LTOrEqualToExpr g = GenPhyOp.compLTOrEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

@Test
public void testDataByteArrayLt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("a"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    GreaterThanExpr g = GenPhyOp.compGreaterThanExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);

}
 

@Test
public void testDataByteArrayLt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("a"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    LessThanExpr g = GenPhyOp.compLessThanExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

@Override
public Map<String, Object> exec(Tuple input) throws IOException {

    TupleFactory tupleFactory = TupleFactory.getInstance();
    ArrayList<Object> objList = new ArrayList<Object>();
    objList.add(new Integer(1));
    objList.add(new Double(1.0));
    objList.add(new Float(1.0));
    objList.add(new String("World!"));
    Tuple tuple = tupleFactory.newTuple(objList);

    BagFactory bagFactory = BagFactory.getInstance();
    DataBag bag = bagFactory.newDefaultBag();
    bag.add(tuple);

    Map<String, Object> mapInMap = new HashMap<String, Object>();
    mapInMap.put("int", new Integer(10));
    mapInMap.put("float", new Float(10.0));

    Map<String, Object> myMap = new HashMap<String, Object>();
    myMap.put("string", new String("Hello"));
    myMap.put("int", new Integer(1));
    myMap.put("long", new Long(1));
    myMap.put("float", new Float(1.0));
    myMap.put("double", new Double(1.0));
    myMap.put("dba", new DataByteArray(new String("bytes").getBytes()));
    myMap.put("map", mapInMap);
    myMap.put("tuple", tuple);
    myMap.put("bag", bag);
    return myMap; 
}
 

@Override
public Tuple exec(Tuple input) throws IOException {
  try {
    DataByteArray data = new DataByteArray(countDisctinct(input, Integer.parseInt(p)).getBytes());
    return mTupleFactory.newTuple(data);
  } catch (ExecException ee) {
    throw ee;
  } catch (Exception e) {
    int errCode = 2106;
    String msg = "Error while computing count in "
        + this.getClass().getSimpleName();
    throw new ExecException(msg, errCode, PigException.BUG, e);
  }
}
 

/**
 * Called to write a bytes. 
 */
@Override
protected void writeBytes(Object datum, org.apache.avro.io.Encoder out)
                                throws IOException {
    if (datum instanceof DataByteArray) {
        out.writeBytes(((DataByteArray) datum).get());
    } else
        throw new RuntimeException("Unsupported type bytes:" + datum.getClass());
}
 

@Test
public void testDataByteArrayGt() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("b"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("a"));
    LTOrEqualToExpr g = GenPhyOp.compLTOrEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);
}
 

@Test
public void testDataByteArrayEq() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("b"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    GTOrEqualToExpr g = GenPhyOp.compGTOrEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertTrue((Boolean)r.result);
}
 

Tuple BackPropConstraint(Tuple outputConstraint, List<Integer> cols,
        LogicalSchema inputSchema, boolean cast) throws ExecException {
    Tuple inputConst = TupleFactory.getInstance().newTuple(
            inputSchema.getFields().size());

    Tuple inputConstraint = new ExampleTuple(inputConst);

    for (int outCol = 0; outCol < outputConstraint.size(); outCol++) {
        int inCol = cols.get(outCol);
        Object outVal = outputConstraint.get(outCol);
        Object inVal = inputConstraint.get(inCol);

        if (inVal == null && outVal != null) {
            // inputConstraint.set(inCol, outVal);
            inputConstraint.set(inCol, (cast) ? new DataByteArray(outVal
                    .toString().getBytes()) : outVal);

        } else {
            if (outVal != null) {
                // unable to back-propagate, due to conflicting column
                // constraints, so give up
                return null;
            }
        }
    }

    return inputConstraint;
}
 

@Test
public void testUnTypedMap() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
    String[] input = {
            "[key#1,key2#2]",
    };

    Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testUnTypedMap", input);

    String query = "a = load '" + tmpDirName + "/testUnTypedMap' as (m:[]);";
    Util.registerMultiLineQuery(pig, query);
    Schema sch = pig.dumpSchema("a");
    assertEquals("Checking expected schema",sch.toString(), "{m: map[]}");
    Iterator<Tuple> it = pig.openIterator("a");

    Assert.assertTrue(it.hasNext());
    Tuple t = it.next();
    Assert.assertTrue(t.size()==1);
    Assert.assertTrue(t.get(0) instanceof Map);
    Assert.assertTrue(((Map)t.get(0)).containsKey("key"));
    Assert.assertTrue(((Map)t.get(0)).containsKey("key2"));
    Assert.assertTrue(((Map)t.get(0)).get("key") instanceof DataByteArray);
    Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
    Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof DataByteArray);
    Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));

    Assert.assertFalse(it.hasNext());
}
 

public static void assertKeyValueEqualsTuple(Key key, Value value,
        Tuple tuple) throws ExecException {
    assertTrue(Arrays.equals(key.getRow().getBytes(),
            ((DataByteArray) tuple.get(0)).get()));
    assertTrue(Arrays.equals(key.getColumnFamily().getBytes(),
            ((DataByteArray) tuple.get(1)).get()));
    assertTrue(Arrays.equals(key.getColumnQualifier().getBytes(),
            ((DataByteArray) tuple.get(2)).get()));
    assertTrue(Arrays.equals(key.getColumnVisibility().getBytes(),
            ((DataByteArray) tuple.get(3)).get()));
    assertEquals(key.getTimestamp(), ((Long) tuple.get(4)).longValue());
    assertTrue(Arrays.equals(value.get(),
            ((DataByteArray) tuple.get(5)).get()));
}
 

/**
 * Read the bytes between start and end into a DataByteArray for inclusion in the return tuple.
 * @param bytes byte array to copy data from
 * @param start starting point to copy from
 * @param end ending point to copy to, exclusive.
 * @return
 */
protected DataByteArray readField(byte[] bytes, int start, int end) {
    if (start == end) {
        return null;
    } else {
        return new DataByteArray(bytes, start, end);
    }
}
 

@Test
public void testReadBytesWritable() throws IOException {
    File inputFile = File.createTempFile("test", ".txt");
    System.err.println("fileName: " + inputFile.getAbsolutePath());
    Path path = new Path("file:///" + inputFile.getAbsolutePath());
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.get(path.toUri(), conf);

    IntWritable key = new IntWritable();
    SequenceFile.Writer writer = null;
    try {
        writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), BytesWritable.class);
        int numRecords = 3;
        for (int i = 0; i < numRecords; i++) {
            key.set(i);
            String val = "" + Math.pow(10, (numRecords - i));
            writer.append(key, new BytesWritable(val.getBytes()));
        }
    } finally {
        IOUtils.closeStream(writer);
    }

    Data data = resetData(pigServer);
    data.set("expected",
            tuple(0L, new DataByteArray("1000.0")),
            tuple(1L, new DataByteArray("100.0")),
            tuple(2L, new DataByteArray("10.0")));

    pigServer.registerQuery(
            "A = LOAD '" + Util.encodeEscape(inputFile.getAbsolutePath()) +
            "' USING org.apache.pig.piggybank.storage.SequenceFileLoader() AS (key:long, val);");
    pigServer.registerQuery("STORE A into 'actual' USING mock.Storage();");

    assertEquals(data.get("expected"), data.get("actual"));

}
 

/**
 * For testing only, do not use directly.
 */
public void setFilter(DataByteArray dba) throws IOException {
    DataInputStream dis = new DataInputStream(new
        ByteArrayInputStream(dba.get()));
    filter = new BloomFilter();
    filter.readFields(dis);
}
 

@Test
public void testLongByteArrays() throws Exception {
  for(int length: new int [] {BinInterSedes.UNSIGNED_BYTE_MAX + 50,
                              BinInterSedes.UNSIGNED_SHORT_MAX + 50} ) {
    // To make sure that offset & length are set correctly for
    // BinInterSedes.SMALLBYTEARRAY and BinInterSedes.BYTEARRAY,
    // create a long bytearray and compare with first or last byte changed

    byte [] ba1 = new byte[length];
    byte [] ba2 = new byte[length];
    Arrays.fill(ba1, 0, length, (byte)'a');
    Arrays.fill(ba2, 0, length, (byte)'a');
    //changing only the last byte
    ba2[length-1] = 'b';
    assertTrue("ByteArray with length: " + length
               + " compare failed with the last byte",
               compareTwoObjectsAsNullableBytesWritables(
                    new DataByteArray(ba1), new DataByteArray(ba2)) != 0);
    //setting back
    ba2[length-1] = 'a';

    //now changing the first byte
    ba2[0] = 'b';
    assertTrue("ByteArray with length: " + length
               + " compare failed with the first byte",
               compareTwoObjectsAsNullableBytesWritables(
                    new DataByteArray(ba1), new DataByteArray(ba2)) != 0);
  }
}
 

@Test
public void testDataByteArrayEq() throws Exception {
    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(new DataByteArray("b"));
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(new DataByteArray("b"));
    GreaterThanExpr g = GenPhyOp.compGreaterThanExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.BYTEARRAY);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean)r.result);
}