Java Code Examples for org.apache.pig.impl.logicalLayer.schema.Schema#getPigSchema()

The following examples show how to use org.apache.pig.impl.logicalLayer.schema.Schema#getPigSchema() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestResourceSchema.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    Schema origSchema = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    ResourceSchema rsSchema = new ResourceSchema(origSchema);
    assertEquals("num fields", aliases.length, rsSchema.getFields().length);
    ResourceSchema.ResourceFieldSchema[] fields = rsSchema.getFields();
    for (int i=0; i<fields.length; i++) {
        assertEquals(fields[i].getName(), aliases[i]);
        assertEquals(fields[i].getType(), types[i]);
    }
    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}
Example 2
Source File: TestResourceSchema.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation2() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}
Example 3
Source File: TestResourceSchema.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
 * Test one-level Pig Schema: multiple fields for a bag
 */
@Test
public void testResourceSchemaWithInvalidPigSchema() 
throws FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    Schema level0 = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    Schema.FieldSchema fld0 = 
        new Schema.FieldSchema("f0", level0, DataType.BAG);
    Schema level1 = new Schema(fld0);
    try {
        Schema.getPigSchema(new ResourceSchema(level1));
        Assert.fail();
    } catch(FrontendException e) {
        assertTrue(e.getErrorCode()==2218);
    }
}
Example 4
Source File: TestResourceSchema.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
 * Test one-level Pig Schema: bag without tuple field
 */
@Test
public void testResourceSchemaWithInvalidPigSchema2() 
throws FrontendException {
    String [] aliases ={"f1"};
    byte[] types = {DataType.INTEGER};
    Schema level0 = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    Schema.FieldSchema fld0 = 
        new Schema.FieldSchema("f0", level0, DataType.BAG);
    Schema level1 = new Schema(fld0);
    try {
        Schema.getPigSchema(new ResourceSchema(level1));
        Assert.fail();
    } catch (FrontendException e) {
        assertTrue(e.getErrorCode()==2218);
    }
}
Example 5
Source File: VespaStorage.java    From vespa with Apache License 2.0 5 votes vote down vote up 
private String createDocumentOperation(Tuple tuple) throws IOException {
    if (tuple == null || tuple.size() == 0) {
        return null;
    }
    if (resourceSchema == null) {
        return null;
    }

    Map<String, Object> fields = TupleTools.tupleMap(resourceSchema, tuple);
    String docId = TupleTools.toString(fields, template);

    Schema schema = Schema.getPigSchema(resourceSchema);
    return VespaDocumentOperation.create(operation, docId, fields, properties, schema);
}
Example 6
Source File: LOLoad.java    From spork with Apache License 2.0 5 votes vote down vote up 
private LogicalSchema getSchemaFromMetaData() throws FrontendException {
    if (getLoadFunc()!=null && getLoadFunc() instanceof LoadMetadata) {
        try {
            ResourceSchema resourceSchema = ((LoadMetadata)loadFunc).getSchema(getFileSpec().getFileName(), new Job(conf));
            Schema oldSchema = Schema.getPigSchema(resourceSchema);
            return Util.translateSchema(oldSchema);
        } catch (IOException e) {
            throw new FrontendException( this, "Cannot get schema from loadFunc " + loadFunc.getClass().getName(), 2245, e);
        }
    }
    return null;
}
Example 7
Source File: JythonFunction.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Override
public Schema outputSchema(Schema input) {
    if(schema != null) {
        return schema;
    } else {
        if(outputSchemaFunc != null) {
            PyFunction pf;
            try {
                pf = JythonScriptEngine.getFunction(scriptFilePath, outputSchemaFunc);
                // this should be a schema function
                PyObject schemaFunctionDef = pf.__findattr__("schemaFunction".intern());
                if(schemaFunctionDef == null) {
                    throw new IllegalStateException("Function: "
                            + outputSchemaFunc + " is not a schema function");
                }
                Object newSchema = ((pf.__call__(Py.java2py(input))).__tojava__(Object.class));
                if (newSchema instanceof ResourceSchema) {
                    return(Schema.getPigSchema((ResourceSchema) newSchema));
                }
                else if (newSchema instanceof Schema) {
                    return (Schema) newSchema;
                }
                else {
                    return Utils.getSchemaFromString(newSchema.toString());
                }
            } catch (IOException ioe) {
                throw new IllegalStateException("Could not find function: "
                    + outputSchemaFunc + "()", ioe);
            }
        } else {
            return new Schema(new Schema.FieldSchema(null, DataType.BYTEARRAY));
        }
    }
}
Example 8
Source File: TestResourceSchema.java    From spork with Apache License 2.0 5 votes vote down vote up 
/**
 * Test that ResourceSchema is correctly with SortInfo
 */
@Test
public void testResourceFlatSchemaCreationWithSortInfo() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
    List<SortColInfo> colList = new ArrayList<SortColInfo>();
    SortColInfo col1 = new SortColInfo("f1", 0, SortColInfo.Order.ASCENDING);
    SortColInfo col2 = new SortColInfo("f1", 1, SortColInfo.Order.DESCENDING);
    colList.add(col1);
    colList.add(col2);
    SortInfo sortInfo = new SortInfo(colList);
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema, sortInfo);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
    assertTrue(rsSchema.getSortKeys()[0]==0);
    assertTrue(rsSchema.getSortKeys()[1]==1);
    assertTrue(rsSchema.getSortKeyOrders()[0]==ResourceSchema.Order.ASCENDING);
    assertTrue(rsSchema.getSortKeyOrders()[1]==ResourceSchema.Order.DESCENDING);
}
Example 9
Source File: TestSchema.java    From spork with Apache License 2.0 4 votes vote down vote up 
@Test
public void testResourceSchemaToSchema() throws ParserException,FrontendException{
    Schema s1 = Utils.getSchemaFromString("b:bag{t:tuple(name:chararray,age:int)}");
    Schema s2 = Schema.getPigSchema(new ResourceSchema(s1));
    assertEquals(s1, s2);
}
Example 10
Source File: TestResourceSchema.java    From spork with Apache License 2.0 4 votes vote down vote up 
/**
 * Test that Pig Schema is correctly created given a
 * ResourceSchema and vice versa. Test also that 
 * TwoLevelAccess flag is set for Pig Schema when needed.
 * @throws IOException 
 */
@Test
public void testToPigSchemaWithTwoLevelAccess() throws IOException {
    ResourceFieldSchema[] level0 = 
        new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("fld0").setType(DataType.CHARARRAY),
            new ResourceFieldSchema()
                .setName("fld1").setType(DataType.DOUBLE),
            new ResourceFieldSchema()
                .setName("fld2").setType(DataType.INTEGER)
    };
           
    ResourceSchema rSchema0 = new ResourceSchema()
        .setFields(level0);
    
    ResourceFieldSchema[] level1 = 
        new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t1").setType(DataType.TUPLE)
                .setSchema(rSchema0)
    };
    
    ResourceSchema rSchema1 = new ResourceSchema()
        .setFields(level1);
    
    ResourceFieldSchema[] level2 = 
        new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t2").setType(DataType.BAG)
                .setSchema(rSchema1)
    };
    
    ResourceSchema origSchema = new ResourceSchema()
        .setFields(level2);        
    
    Schema pSchema = Schema.getPigSchema(origSchema);
            
    assertTrue(!CheckTwoLevelAccess(pSchema));
            
    assertTrue(ResourceSchema.equals(origSchema, new ResourceSchema(pSchema)));
}