Java Code Examples for org.apache.pig.newplan.logical.relational.LogicalSchema#addField()

The following examples show how to use org.apache.pig.newplan.logical.relational.LogicalSchema#addField() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: Util.java From spork with Apache License 2.0

6 votes

public static LogicalSchema translateSchema(Schema schema) {
    if (schema == null) {
        return null;
    }

    LogicalSchema s2 = new LogicalSchema();
    List<Schema.FieldSchema> ll = schema.getFields();
    for (Schema.FieldSchema f: ll) {
        LogicalSchema.LogicalFieldSchema f2 =
            new LogicalSchema.LogicalFieldSchema(f.alias, translateSchema(f.schema), f.type);

        s2.addField(f2);
    }

    return s2;
}

Example 2

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testLoadEqualityDifferentFuncSpecCtorArgs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    LOLoad load2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load2);

    assertFalse(load1.isEqual(load2));
}

Example 3

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testLoadEqualityDifferentNumFuncSpecCstorArgs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    LOLoad load3 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), "x")), aschema1, lp, conf);
    lp.add(load3);

    assertFalse(load1.isEqual(load3));
}

Example 4

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testLoadEqualityDifferentFunctionNames() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different function names in FuncSpec
    LOLoad load4 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load4);

    assertFalse(load1.isEqual(load4));
}

Example 5

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testLoadEqualityDifferentFileName() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different file name
    LOLoad load5 = newLOLoad(new FileSpec("/def",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load5);

    assertFalse(load1.isEqual(load5));
}

Example 6

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testRelationalEqualityDifferentSchema() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different schema
    LogicalSchema aschema2 = new LogicalSchema();
    aschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.CHARARRAY));

    LOLoad load6 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema2, lp,
            conf);
    lp.add(load6);

    assertFalse(load1.isEqual(load6));
}

Example 7

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

6 votes

@Test
public void testRelationalEqualityOneNullOneNotNullSchema() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Test that one with schema and one without breaks equality
    LOLoad load9 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp, conf);
    lp.add(load9);

    assertFalse(load1.isEqual(load9));
}

Example 8

Source File: StreamTypeCastInserter.java From spork with Apache License 2.0

5 votes

protected LogicalSchema determineSchema(LogicalRelationalOperator op)
        throws FrontendException {
    LogicalSchema determinedSchema = new LogicalSchema();
    for (int i = 0; i < op.getSchema().size(); i++) {
        determinedSchema.addField(new LogicalFieldSchema(
                null,
                null,
                DataType.BYTEARRAY));
    }
    return determinedSchema;
}

Example 9

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

5 votes

@Test
public void testFilterDifferentPredicates() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
    ProjectExpression fy1 = new ProjectExpression(fp1, 0, 1, null);
    ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0));
    new EqualExpression(fp1, fy1, fc1);
    LOFilter D1 = new LOFilter(lp, fp1);
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    D1.neverUseForRealSetSchema(cschema);
    lp.add(D1);

    LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
    ProjectExpression fy2 = new ProjectExpression(fp2, 0, 1, null);
    ConstantExpression fc2 = new ConstantExpression(fp2,
            new Integer(1));
    new EqualExpression(fp2, fy2, fc2);
    LOFilter D2 = new LOFilter(lp, fp2);
    D2.neverUseForRealSetSchema(cschema);
    lp.add(D2);

    assertFalse(D1.isEqual(D2));
}

Example 10

Source File: UserFuncExpression.java From spork with Apache License 2.0

4 votes

@Override
public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;

    LogicalSchema inputSchema = new LogicalSchema();
    List<Operator> succs = plan.getSuccessors(this);

    if (succs!=null) {
        for(Operator lo : succs){
            if (((LogicalExpression)lo).getFieldSchema()==null) {
                inputSchema = null;
                break;
            }
            inputSchema.addField(((LogicalExpression)lo).getFieldSchema());
        }
    }

    if (lazilyInitializeInvokerFunction) {
        initializeInvokerFunction();
    }

    // Since ef only set one time, we never change its value, so we can optimize it by instantiate only once.
    // This significantly optimize the performance of frontend (PIG-1738)
    if (ef==null) {
        ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
    }

    ef.setUDFContextSignature(signature);
    Properties props = UDFContext.getUDFContext().getUDFProperties(ef.getClass());
    Schema translatedInputSchema = Util.translateSchema(inputSchema);
    if(translatedInputSchema != null) {
        props.put("pig.evalfunc.inputschema."+signature, translatedInputSchema);
    }
    // Store inputSchema into the UDF context
    ef.setInputSchema(translatedInputSchema);

    Schema udfSchema = ef.outputSchema(translatedInputSchema);
    if (udfSchema != null && udfSchema.size() > 1) {
        throw new FrontendException("Given UDF returns an improper Schema. Schema should only contain one field of a Tuple, Bag, or a single type. Returns: " + udfSchema);
    }

    //TODO appendability should come from a setting
    SchemaTupleFrontend.registerToGenerateIfPossible(translatedInputSchema, false, GenContext.UDF);
    SchemaTupleFrontend.registerToGenerateIfPossible(udfSchema, false, GenContext.UDF);

    if (udfSchema != null) {
        Schema.FieldSchema fs;
        if(udfSchema.size() == 0) {
            fs = new Schema.FieldSchema(null, null, DataType.findType(ef.getReturnType()));
        } else if(udfSchema.size() == 1) {
            fs = new Schema.FieldSchema(udfSchema.getField(0));
        } else {
            fs = new Schema.FieldSchema(null, udfSchema, DataType.TUPLE);
        }
        fieldSchema = Util.translateFieldSchema(fs);
        fieldSchema.normalize();
    } else {
        fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.findType(ef.getReturnType()));
    }
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    return fieldSchema;
}

Example 11

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

4 votes

@Test
public void testJoinDifferentJoinTypes() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // A = load
    LogicalSchema jaschema2 = new LogicalSchema();
    jaschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema2, lp, conf);
    lp.add(A2);

    // B = load
    LogicalSchema jbschema2 = new LogicalSchema();
    jbschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B2 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema2, lp, conf);
    lp.add(B2);

    // C = join
    LogicalSchema jcschema2 = new LogicalSchema();
    jcschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan2 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan2, 0, 0, null);
    LogicalExpressionPlan bprojplan2 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan2, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm2 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm2.put(0, aprojplan2);
    mm2.put(1, bprojplan2);
    LOJoin C2 = new LOJoin(lp, mm2, JOINTYPE.SKEWED, new boolean[] { true, true });
    C2.neverUseForRealSetSchema(jcschema2);
    lp.add(C2);
    lp.connect(A2, C2);
    lp.connect(B2, C2);

    assertFalse(C1.isEqual(C2));
}

Example 12

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

4 votes

@Test
public void testJoinDifferentInner() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // Test different inner status
    // A = load
    LogicalSchema jaschema3 = new LogicalSchema();
    jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A3 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema3, lp, conf);
    lp.add(A3);

    // B = load
    LogicalSchema jbschema3 = new LogicalSchema();
    jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B3 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema3, lp, conf);
    lp.add(B3);

    // C = join
    LogicalSchema jcschema3 = new LogicalSchema();
    jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan3 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan3, 0, 0, null);
    LogicalExpressionPlan bprojplan3 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan3, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm3 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm3.put(0, aprojplan3);
    mm3.put(1, bprojplan3);
    LOJoin C3 = new LOJoin(lp, mm3, JOINTYPE.HASH, new boolean[] { true, false });
    C3.neverUseForRealSetSchema(jcschema3);
    lp.add(C3);
    lp.connect(A3, C3);
    lp.connect(B3, C3);

    assertFalse(C1.isEqual(C3));
}

Example 13

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

4 votes

@Test
public void testJoinDifferentNumInputs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // A = load
    LogicalSchema jaschema5 = new LogicalSchema();
    jaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A5 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema5, lp, conf);
    lp.add(A5);

    // B = load
    LogicalSchema jbschema5 = new LogicalSchema();
    jbschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B5 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema5, lp, conf);
    lp.add(B5);

    // Beta = load
    LogicalSchema jbetaschema5 = new LogicalSchema();
    jbetaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad Beta5 = newLOLoad(new FileSpec("/ghi",
            new FuncSpec("PigStorage", "\t")), jbetaschema5, lp, conf);
    lp.add(Beta5);

    // C = join
    LogicalSchema jcschema5 = new LogicalSchema();
    jcschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan5 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan5, 0, 0, null);
    LogicalExpressionPlan bprojplan5 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan5, 1, 0, null);
    LogicalExpressionPlan betaprojplan5 = new LogicalExpressionPlan();
    new ProjectExpression(betaprojplan5, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm5 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm5.put(0, aprojplan5);
    mm5.put(1, bprojplan5);
    mm5.put(2, betaprojplan5);
    LOJoin C5 = new LOJoin(lp, mm5, JOINTYPE.HASH, new boolean[] { true, true });
    C5.neverUseForRealSetSchema(jcschema5);
    lp.add(C5);
    lp.connect(A5, C5);
    lp.connect(B5, C5);
    lp.connect(Beta5, C5);

    assertFalse(C1.isEqual(C5));
}

Example 14

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

4 votes

@Test
public void testJoinDifferentJoinKeys() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    // Test different join keys
    LogicalSchema jaschema6 = new LogicalSchema();
    jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A6 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema6, lp, conf);
    lp.add(A6);

    // B = load
    LogicalSchema jbschema6 = new LogicalSchema();
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B6 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, conf);
    lp.add(B6);

    // C = join
    LogicalSchema jcschema6 = new LogicalSchema();
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan6, 0, 0, null);
    LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan6, 1, 0, null);
    LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
    new ProjectExpression(b2projplan6, 1, 1, null);
    MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm6.put(0, aprojplan6);
    mm6.put(1, bprojplan6);
    mm6.put(1, b2projplan6);
    LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] { true, true });
    C6.neverUseForRealSetSchema(jcschema6);
    lp.add(C6);
    lp.connect(A6, C6);
    lp.connect(B6, C6);

    LogicalSchema jaschema7 = new LogicalSchema();
    jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A7 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema7, lp, conf);
    lp.add(A7);

    // B = load
    LogicalSchema jbschema7 = new LogicalSchema();
    jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B7 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema7, lp, conf);
    lp.add(B7);

    // C = join
    LogicalSchema jcschema7 = new LogicalSchema();
    jcschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan7 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan7, 0, 0, null);
    LogicalExpressionPlan bprojplan7 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan7, 1, 1, null);
    LogicalExpressionPlan b2projplan7 = new LogicalExpressionPlan();
    new ProjectExpression(b2projplan7, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm7 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm7.put(0, aprojplan7);
    mm7.put(1, bprojplan7);
    mm7.put(1, b2projplan7);
    LOJoin C7 = new LOJoin(lp, mm7, JOINTYPE.HASH, new boolean[] { true, true });
    C7.neverUseForRealSetSchema(jcschema7);
    lp.add(C7);
    lp.connect(A7, C7);
    lp.connect(B7, C7);

    assertFalse(C6.isEqual(C7));
}

Example 15

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

4 votes

@Test
public void testJoinDifferentNumJoinKeys() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    // Test different join keys
    LogicalSchema jaschema6 = new LogicalSchema();
    jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A6 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema6, lp, conf);
    lp.add(A6);

    // B = load
    LogicalSchema jbschema6 = new LogicalSchema();
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B6 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, conf);
    lp.add(B6);

    // C = join
    LogicalSchema jcschema6 = new LogicalSchema();
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan6, 0, 0, null);
    LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan6, 1, 0, null);
    LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
    new ProjectExpression(b2projplan6, 1, 1, null);
    MultiMap<Integer, LogicalExpressionPlan> mm6 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm6.put(0, aprojplan6);
    mm6.put(1, bprojplan6);
    mm6.put(1, b2projplan6);
    LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] { true, true });
    C6.neverUseForRealSetSchema(jcschema6);
    lp.add(C6);
    lp.connect(A6, C6);
    lp.connect(B6, C6);

    // Test different different number of join keys
    LogicalSchema jaschema8 = new LogicalSchema();
    jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A8 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema8, lp, conf);
    lp.add(A8);

    // B = load
    LogicalSchema jbschema8 = new LogicalSchema();
    jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
    LOLoad B8 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema8, lp, conf);
    lp.add(B8);

    // C = join
    LogicalSchema jcschema8 = new LogicalSchema();
    jcschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan8 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan8, 0, 0, null);
    LogicalExpressionPlan bprojplan8 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan8, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm8 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm8.put(0, aprojplan8);
    mm8.put(1, bprojplan8);
    LOJoin C8 = new LOJoin(lp, mm8, JOINTYPE.HASH, new boolean[] { true, true });
    C8.neverUseForRealSetSchema(jcschema8);
    lp.add(C8);
    lp.connect(A8, C8);
    lp.connect(B8, C8);

    assertFalse(C6.isEqual(C8));
}

Example 16

Source File: TestNewPlanOperatorPlan.java From spork with Apache License 2.0

4 votes

@Test
public void testRelationalSameOpDifferentPreds() throws FrontendException {
    LogicalPlan lp1 = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp1,
            conf);
    lp1.add(A1);

    LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
    ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
    ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0));
    new EqualExpression(fp1, fy1, fc1);
    LOFilter D1 = new LOFilter(lp1, fp1);
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    D1.neverUseForRealSetSchema(cschema);
    lp1.add(D1);
    lp1.connect(A1, D1);

    LogicalPlan lp2 = new LogicalPlan();
    LOLoad A2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp2,
            conf);
    lp2.add(A2);

    LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
    ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
    ConstantExpression fc2 = new ConstantExpression(fp2,
            new Integer(0));
    new EqualExpression(fp2, fy2, fc2);
    LOFilter D2 = new LOFilter(lp2, fp2);
    D2.neverUseForRealSetSchema(cschema);
    lp2.add(D2);
    lp2.connect(A2, D2);

    assertTrue(D1.isEqual(D2));
}

Example 17

Source File: TestNewPlanListener.java From spork with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
    // Build a plan that looks like it has just been transformed
    // It is roughly the logical plan for
    // A = load 'bla' as (x);
    // B = load 'morebla' as (y);
    // C = join A on x, B on y;
    // D = filter C by y > 0;
    // The plan is built with the filter having been pushed above the join
    // but the listners not yet having been called.
    // A = load
    lp = new LogicalPlan();
    LogicalSchema aschema = new LogicalSchema();
    aschema.addField(new LogicalSchema.LogicalFieldSchema(
        "x", null, DataType.INTEGER));
    aschema.getField(0).uid = 1;
    LOLoad A = new LOLoad(null, lp);
    A.neverUseForRealSetSchema(aschema);
    lp.add(A);

    // B = load
    LogicalSchema bschema = new LogicalSchema();
    bschema.addField(new LogicalSchema.LogicalFieldSchema(
        "y", null, DataType.INTEGER));
    bschema.getField(0).uid = 2;
    LOLoad B = new LOLoad(null, lp);
    B.neverUseForRealSetSchema(bschema);
    lp.add(B);

    // C = join
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
        "x", null, DataType.INTEGER));
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
        "y", null, DataType.INTEGER));
    cschema.getField(0).uid = 1;
    cschema.getField(1).uid = 2;

    MultiMap<Integer, LogicalExpressionPlan> mm =
        new MultiMap<Integer, LogicalExpressionPlan>();
    LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});

    LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
    ProjectExpression x = new ProjectExpression(aprojplan, 0, 0, C);
    x.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
            DataType.INTEGER, 1));
    LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
    ProjectExpression y = new ProjectExpression(bprojplan, 1, 0, C);
    y.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
            DataType.INTEGER, 2));
    mm.put(0, aprojplan);
    mm.put(1, bprojplan);

    C.neverUseForRealSetSchema(cschema);
    // Don't add it to the plan quite yet

    // D = filter
    LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
    LOFilter D = new LOFilter(lp, filterPlan);
    ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
    fy.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null,
            DataType.INTEGER, 2));
    ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
    new EqualExpression(filterPlan, fy, fc);

    D.neverUseForRealSetSchema(cschema);
    // Connect D to B, since the transform has happened.
    lp.add(D);
    lp.connect(B, D);

    // Now add in C, connected to A and D.
    lp.add(C);
    lp.connect(A, C);
    lp.connect(D, C);

    changedPlan = new LogicalPlan();
    changedPlan.add(D);
    changedPlan.add(C);
    changedPlan.connect(D, C);
}

Example 18

Source File: TestNewPlanLogToPhyTranslationVisitor.java From spork with Apache License 2.0

4 votes

@Test    
public void testForeachPlan2() throws Exception {
    String query = ("a = load 'd.txt' as (id, c:bag{t:(s,v)});" +
    "b = foreach a generate id, flatten(c);" +        
    "store b into 'empty';");  
    
    LogicalPlan newLogicalPlan = buildPlan(query);
    LogicalRelationalOperator ld =  (LogicalRelationalOperator)newLogicalPlan.getSources().get(0);
    LogicalRelationalOperator fe = (LogicalRelationalOperator)newLogicalPlan.getSuccessors(ld).get(0);
    LogicalSchema ls = fe.getSchema();
    assertEquals(1, ls.getField(0).uid);
    assertEquals(4, ls.getField(1).uid);
    assertEquals(5, ls.getField(2).uid);
    
    LogicalSchema expected = new LogicalSchema();
    expected.addField(new LogicalFieldSchema("id", null, DataType.BYTEARRAY));
    expected.addField(new LogicalFieldSchema("s", null, DataType.BYTEARRAY));
    expected.addField(new LogicalFieldSchema("v", null, DataType.BYTEARRAY));
    assertTrue(expected.isEqual(ls));
    
    
    PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
    
    assertEquals(phyPlan.size(), 3);
    assertEquals(phyPlan.getLeaves().get(0).getClass(), POStore.class);
    POForEach foreach = (POForEach)phyPlan.getSuccessors(phyPlan.getRoots().get(0)).get(0);
    
    assertEquals(foreach.getInputPlans().size(), 2);
    
    PhysicalPlan inner = foreach.getInputPlans().get(0);
    assertEquals(inner.size(), 1);
    POProject prj = (POProject)inner.getRoots().get(0);
    assertEquals(prj.getColumn(), 0);
    assertEquals(prj.getInputs(), null);
    
    inner = foreach.getInputPlans().get(1);
    assertEquals(inner.size(), 1);
    prj = (POProject)inner.getRoots().get(0);
    assertEquals(prj.getColumn(), 1);
    assertEquals(prj.getInputs(), null);
    Boolean[] flat = foreach.getToBeFlattened().toArray(new Boolean[0]);
    assertFalse(flat[0]);
    assertTrue(flat[1]);
}