Java Code Examples for org.apache.pig.impl.logicalLayer.schema.Schema#mergeSchema()

The following examples show how to use org.apache.pig.impl.logicalLayer.schema.Schema#mergeSchema() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestSchema.java From spork with Apache License 2.0

6 votes

@Test(expected = SchemaMergeException.class)
public void testMergeDifferentSize2() throws Throwable {
    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1c", DataType.LONG));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2c", DataType.INTEGER));
    list2.add(new FieldSchema("2d", DataType.MAP));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                         schema2,
                                         true,
                                         false,
                                         false);
}

Example 2

Source File: TestSchema.java From spork with Apache License 2.0

5 votes

@Test
// See PIG-730
public void testMergeSchemaWithTwoLevelAccess1() throws Exception {
    // Generate two schemas
    Schema s1 = Utils.getSchemaFromString("a:{t:(a0:int, a1:int)}");
    Schema s2 = Utils.getSchemaFromString("b:{t:(b0:int, b1:int)}");
    s1.getField(0).schema.setTwoLevelAccessRequired(true);
    s2.getField(0).schema.setTwoLevelAccessRequired(true);
    Schema s3 = Schema.mergeSchema(s1, s2, true);
    assertTrue(s3.getField(0).schema.isTwoLevelAccessRequired());
}

Example 3

Source File: TestSchema.java From spork with Apache License 2.0

5 votes

@Test
// See PIG-730
public void testMergeSchemaWithTwoLevelAccess() throws Exception {
    // Generate two schemas
    Schema s1 = Utils.getSchemaFromString("a:{t:(a0:int, a1:int)}");
    Schema s2 = Utils.getSchemaFromString("b:{t:(b0:int, b1:int)}");
    s1.getField(0).schema.setTwoLevelAccessRequired(true);
    s1.getField(0).schema.setTwoLevelAccessRequired(false);
    Schema s3 = Schema.mergeSchema(s1, s2, true);
    assertEquals(s3, s2);
}

Example 4

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeNullSchemas1() throws Throwable {

    List<FieldSchema> innerList2 = new ArrayList<FieldSchema>();
    innerList2.add(new FieldSchema("22a", DataType.DOUBLE));
    innerList2.add(new FieldSchema(null, DataType.LONG));

    Schema innerSchema2 = new Schema(innerList2);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", null));
    list1.add(new FieldSchema("1c", DataType.LONG));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", innerSchema2));
    list2.add(new FieldSchema("2c", DataType.INTEGER));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                             schema2,
                                             true,
                                             false,
                                             true);


    // Generate expected schema

    List<FieldSchema> expectedList = new ArrayList<FieldSchema>();
    expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY));
    expectedList.add(new FieldSchema("2b", null));
    expectedList.add(new FieldSchema("2c", DataType.LONG));

    Schema expected = new Schema(expectedList);

    // Compare
    assertTrue(Schema.equals(mergedSchema, expected, false, false));
}

Example 5

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeNullSchemas2() throws Throwable {
    // Inner of inner schema
    Schema innerInner = new Schema(new ArrayList<FieldSchema>());

    // Generate two schemas
    List<FieldSchema> innerList1 = new ArrayList<FieldSchema>();
    innerList1.add(new FieldSchema("11a", innerInner));
    innerList1.add(new FieldSchema("11b", DataType.FLOAT));

    List<FieldSchema> innerList2 = new ArrayList<FieldSchema>();
    innerList2.add(new FieldSchema("22a", null));
    innerList2.add(new FieldSchema(null, DataType.LONG));

    Schema innerSchema1 = new Schema(innerList1);
    Schema innerSchema2 = new Schema(innerList2);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", innerSchema1));
    list1.add(new FieldSchema("1c", DataType.LONG));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", innerSchema2));
    list2.add(new FieldSchema("2c", DataType.INTEGER));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                             schema2,
                                             true,
                                             false,
                                             true);


    // Generate expected schema
    List<FieldSchema> expectedInnerList = new ArrayList<FieldSchema>();
    expectedInnerList.add(new FieldSchema("22a", null));
    expectedInnerList.add(new FieldSchema("11b", DataType.FLOAT));

    Schema expectedInner = new Schema(expectedInnerList);

    List<FieldSchema> expectedList = new ArrayList<FieldSchema>();
    expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY));
    expectedList.add(new FieldSchema("2b", expectedInner));
    expectedList.add(new FieldSchema("2c", DataType.LONG));

    Schema expected = new Schema(expectedList);

    // Compare
    assertTrue(Schema.equals(mergedSchema, expected, false, false));
}

Example 6

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeDifferentSize1() throws Throwable {

    // Generate two schemas
    List<FieldSchema> innerList1 = new ArrayList<FieldSchema>();
    innerList1.add(new FieldSchema("11a", DataType.INTEGER));
    innerList1.add(new FieldSchema("11b", DataType.FLOAT));
    innerList1.add(new FieldSchema("11c", DataType.CHARARRAY));

    List<FieldSchema> innerList2 = new ArrayList<FieldSchema>();
    innerList2.add(new FieldSchema("22a", DataType.DOUBLE));
    innerList2.add(new FieldSchema(null, DataType.LONG));

    Schema innerSchema1 = new Schema(innerList1);
    Schema innerSchema2 = new Schema(innerList2);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", innerSchema1));
    list1.add(new FieldSchema("1c", DataType.LONG));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", innerSchema2));
    list2.add(new FieldSchema("2c", DataType.INTEGER));
    list2.add(new FieldSchema("2d", DataType.MAP));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                             schema2,
                                             true,
                                             true,
                                             false);


    // Generate expected schema
    List<FieldSchema> expectedInnerList = new ArrayList<FieldSchema>();
    expectedInnerList.add(new FieldSchema("22a", DataType.DOUBLE));
    expectedInnerList.add(new FieldSchema("11b", DataType.FLOAT));
    expectedInnerList.add(new FieldSchema("11c", DataType.CHARARRAY));

    Schema expectedInner = new Schema(expectedInnerList);

    List<FieldSchema> expectedList = new ArrayList<FieldSchema>();
    expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY));
    expectedList.add(new FieldSchema("2b", expectedInner));
    expectedList.add(new FieldSchema("2c", DataType.LONG));
    expectedList.add(new FieldSchema("2d", DataType.MAP));

    Schema expected = new Schema(expectedList);

    // Compare
    assertTrue(Schema.equals(mergedSchema, expected, false, false));
}

Example 7

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeMismatchType1() throws Throwable {

    // Generate two schemas
    List<FieldSchema> innerList1 = new ArrayList<FieldSchema>();
    innerList1.add(new FieldSchema("11a", DataType.CHARARRAY));
    innerList1.add(new FieldSchema("11b", DataType.FLOAT));

    List<FieldSchema> innerList2 = new ArrayList<FieldSchema>();
    innerList2.add(new FieldSchema("22a", DataType.DOUBLE));
    innerList2.add(new FieldSchema(null, DataType.LONG));

    Schema innerSchema1 = new Schema(innerList1);
    Schema innerSchema2 = new Schema(innerList2);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", innerSchema1));
    list1.add(new FieldSchema("1c", DataType.MAP));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", innerSchema2));
    list2.add(new FieldSchema("2c", DataType.INTEGER));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                             schema2,
                                             true,
                                             false,
                                             true);


    // Generate expected schema
    List<FieldSchema> expectedInnerList = new ArrayList<FieldSchema>();
    expectedInnerList.add(new FieldSchema("22a", DataType.BYTEARRAY));
    expectedInnerList.add(new FieldSchema("11b", DataType.FLOAT));

    Schema expectedInner = new Schema(expectedInnerList);

    List<FieldSchema> expectedList = new ArrayList<FieldSchema>();
    expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY));
    expectedList.add(new FieldSchema("2b", expectedInner));
    expectedList.add(new FieldSchema("2c", DataType.BYTEARRAY));

    Schema expected = new Schema(expectedList);

    // Compare
    assertTrue(Schema.equals(mergedSchema, expected, false, false));
}

Example 8

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeMismatchType2() throws Throwable {

    // Generate two schemas
    List<FieldSchema> innerList1 = new ArrayList<FieldSchema>();
    innerList1.add(new FieldSchema("11a", DataType.CHARARRAY));
    innerList1.add(new FieldSchema("11b", DataType.FLOAT));

    Schema innerSchema1 = new Schema(innerList1);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", innerSchema1));
    list1.add(new FieldSchema("1c", DataType.MAP));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2c", DataType.INTEGER));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                             schema2,
                                             true,
                                             false,
                                             true);


    // Generate expected schema
    List<FieldSchema> expectedList = new ArrayList<FieldSchema>();
    expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY));
    expectedList.add(new FieldSchema("2b", DataType.TUPLE));
    expectedList.add(new FieldSchema("2c", DataType.BYTEARRAY));

    Schema expected = new Schema(expectedList);

    // Compare
    assertTrue(Schema.equals(mergedSchema, expected, false, false));
}

Example 9

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeMismatchType3()  {

  // Generate two schemas
    List<FieldSchema> innerList1 = new ArrayList<FieldSchema>();
    innerList1.add(new FieldSchema("11a", DataType.CHARARRAY));
    innerList1.add(new FieldSchema("11b", DataType.FLOAT));

    List<FieldSchema> innerList2 = new ArrayList<FieldSchema>();
    innerList2.add(new FieldSchema("22a", DataType.DOUBLE));
    innerList2.add(new FieldSchema(null, DataType.LONG));

    Schema innerSchema1 = new Schema(innerList1);
    Schema innerSchema2 = new Schema(innerList2);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.BYTEARRAY));
    list1.add(new FieldSchema("1b", innerSchema1));
    list1.add(new FieldSchema("1c", DataType.MAP));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.BYTEARRAY));
    list2.add(new FieldSchema("2b", innerSchema2));
    list2.add(new FieldSchema("2c", DataType.MAP));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge

    try {
        Schema mergedSchema = Schema.mergeSchema(schema1,
                                                 schema2,
                                                 true,
                                                 false,
                                                 false);
        fail("Expect error here!");
    } catch (SchemaMergeException e) {
        // good
    }
}

Example 10

Source File: TestSchema.java From spork with Apache License 2.0

4 votes

@Test
public void testMergeDifferentSizeAndTypeMismatch1() throws Throwable {

    // Generate two schemas
    List<FieldSchema> innerList1 = new ArrayList<FieldSchema>();
    innerList1.add(new FieldSchema("11a", DataType.INTEGER));
    innerList1.add(new FieldSchema("11b", DataType.FLOAT));
    innerList1.add(new FieldSchema("11c", DataType.CHARARRAY));

    List<FieldSchema> innerList2 = new ArrayList<FieldSchema>();
    innerList2.add(new FieldSchema("22a", DataType.CHARARRAY));
    innerList2.add(new FieldSchema(null, DataType.LONG));

    Schema innerSchema1 = new Schema(innerList1);
    Schema innerSchema2 = new Schema(innerList2);

    List<FieldSchema> list1 = new ArrayList<FieldSchema>();
    list1.add(new FieldSchema("1a", DataType.INTEGER));
    list1.add(new FieldSchema("1b", innerSchema1));
    list1.add(new FieldSchema("1c", DataType.LONG));

    List<FieldSchema> list2 = new ArrayList<FieldSchema>();
    list2.add(new FieldSchema("2a", DataType.CHARARRAY));
    list2.add(new FieldSchema("2b", innerSchema2));
    list2.add(new FieldSchema("2c", DataType.INTEGER));
    list2.add(new FieldSchema("2d", DataType.MAP));

    Schema schema1 = new Schema(list1);
    Schema schema2 = new Schema(list2);

    // Merge
    Schema mergedSchema = Schema.mergeSchema(schema1,
                                             schema2,
                                             true,
                                             true,
                                             true);


    // Generate expected schema
    List<FieldSchema> expectedInnerList = new ArrayList<FieldSchema>();
    expectedInnerList.add(new FieldSchema("22a", DataType.BYTEARRAY));
    expectedInnerList.add(new FieldSchema("11b", DataType.FLOAT));
    expectedInnerList.add(new FieldSchema("11c", DataType.CHARARRAY));

    Schema expectedInner = new Schema(expectedInnerList);

    List<FieldSchema> expectedList = new ArrayList<FieldSchema>();
    expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY));
    expectedList.add(new FieldSchema("2b", expectedInner));
    expectedList.add(new FieldSchema("2c", DataType.LONG));
    expectedList.add(new FieldSchema("2d", DataType.MAP));

    Schema expected = new Schema(expectedList);

    // Compare
    assertTrue(Schema.equals(mergedSchema, expected, false, false));
}