Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#javaIntObjectInspector()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#javaIntObjectInspector() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: PassiveAggressiveUDTFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testPA1TrainWithParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 0.1");
    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.1000000f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(3).get(), 1e-5f);
}

Example 2

Source File: GeneralClassifierUDTFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testInspectOptimizerOptions() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        "-opt adam -reg l1 -inspect_opts");

    try {
        udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
        Assert.fail("should not come here");
    } catch (UDFArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("adam"));
    }
}

Example 3

Source File: PassiveAggressiveUDTFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testPA2EtaWithParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA2();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 3.0");

    /* do initialize() with aggressiveness parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});
    float loss = 0.1f;

    PredictionResult margin1 = new PredictionResult(0.5f).squaredNorm(0.05f);
    float expectedLearningRate1 = 0.4615384f;
    assertEquals(expectedLearningRate1, udtf.eta(loss, margin1), 1e-5f);

    PredictionResult margin2 = new PredictionResult(0.5f).squaredNorm(0.01f);
    float expectedLearningRate2 = 0.5660377f;
    assertEquals(expectedLearningRate2, udtf.eta(loss, margin2), 1e-5f);
}

Example 4

Source File: GeneralClassifierUDTFTest.java From incubator-hivemall with Apache License 2.0

6 votes

@Test
public void testNoOptions() throws Exception {
    List<String> x = Arrays.asList("1:-2", "2:-1");
    int y = 0;

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI});

    udtf.process(new Object[] {x, y});

    udtf.finalizeTraining();

    float score = udtf.predict(udtf.parseFeatures(x));
    int predicted = score > 0.f ? 1 : 0;
    Assert.assertTrue(y == predicted);
}

Example 5

Source File: FeatureUDFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testIntInt() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(
        new GenericUDF.DeferredObject[] {new DeferredJavaObject(1), new DeferredJavaObject(2)});

    Assert.assertEquals("1:2", ret.toString());
}

Example 6

Source File: UDAFToOrderedMapTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testTopK() throws Exception {
    TopKOrderedMapEvaluator evaluator = new TopKOrderedMapEvaluator();
    TopKOrderedMapEvaluator.MapAggregationBuffer agg =
            (TopKOrderedMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();

    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaIntObjectInspector};

    final double[] keys = new double[] {0.7, 0.5, 0.8};
    final String[] values = new String[] {"banana", "apple", "candy"};
    int size = 2;

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < keys.length; i++) {
        evaluator.iterate(agg, new Object[] {keys[i], values[i], size});
    }

    Map<Object, Object> res = evaluator.terminate(agg);
    Object[] sortedValues = res.values().toArray();

    Assert.assertEquals(size, sortedValues.length);
    Assert.assertEquals("candy", sortedValues[0]);
    Assert.assertEquals("banana", sortedValues[1]);

    evaluator.close();
}

Example 7

Source File: PassiveAggressiveUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testInitialize() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, intOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, intOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}

Example 8

Source File: GeneralClassifierUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

private <T> void testFeature(@Nonnull List<T> x, @Nonnull ObjectInspector featureOI,
        @Nonnull Class<T> featureClass, @Nonnull Class<?> modelFeatureClass) throws Exception {
    int y = 0;

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector featureListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(featureOI);

    udtf.initialize(new ObjectInspector[] {featureListOI, valueOI});

    final List<Object> modelFeatures = new ArrayList<Object>();
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object input) throws HiveException {
            Object[] forwardMapObj = (Object[]) input;
            modelFeatures.add(forwardMapObj[0]);
        }
    });

    udtf.process(new Object[] {x, y});

    udtf.close();

    Assert.assertFalse(modelFeatures.isEmpty());
    for (Object modelFeature : modelFeatures) {
        Assert.assertEquals("All model features must have same type", modelFeatureClass,
            modelFeature.getClass());
    }
}

Example 9

Source File: UDAFToOrderedListTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testVKMapTop2() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final int[] keys = new int[] {5, 3, 4, 2, 3};
    final String[] values = new String[] {"apple", "banana", "candy", "donut", "egg"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(2, map.size());

    Assert.assertEquals(5, map.get("apple"));
    Assert.assertEquals(4, map.get("candy"));
}

Example 10

Source File: PerceptronUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testInitialize() throws UDFArgumentException {
    PerceptronUDTF udtf = new PerceptronUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, intOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, intOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}

Example 11

Source File: PassiveAggressiveUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testTrain() throws HiveException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights by List<Object> */
    List<Integer> features1 = new ArrayList<Integer>();
    features1.add(1);
    features1.add(2);
    features1.add(3);
    udtf.train(features1, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);

    /* train weights by Object[] */
    List<?> features2 = (List<?>) intListOI.getList(new Object[] {3, 4, 5});
    udtf.train(features2, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.5555555f, udtf.model.get(3).get(), 1e-5f);
    assertEquals(0.2222222f, udtf.model.get(4).get(), 1e-5f);
    assertEquals(0.2222222f, udtf.model.get(5).get(), 1e-5f);
}

Example 12

Source File: GeneralClassifierUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test(expected = UDFArgumentException.class)
public void testUnsupportedLossFunction() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-loss UnsupportedLoss");

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
}

Example 13

Source File: PassiveAggressiveUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test
public void testPA1EtaDefaultParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    float loss = 0.1f;

    PredictionResult margin = new PredictionResult(0.5f).squaredNorm(0.05f);
    float expectedLearningRate = 1.0f;
    assertEquals(expectedLearningRate, udtf.eta(loss, margin), 1e-5f);
}

Example 14

Source File: GeneralClassifierUDTFTest.java From incubator-hivemall with Apache License 2.0

5 votes

@Test(expected = UDFArgumentException.class)
public void testUnsupportedOptimizer() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-opt UnsupportedOpt");

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
}

Example 15

Source File: GeneralClassifierUDTFTest.java From incubator-hivemall with Apache License 2.0

4 votes

private void run(@Nonnull String options) throws Exception {
    println(options);

    ArrayList<List<String>> samplesList = new ArrayList<List<String>>();
    samplesList.add(Arrays.asList("1:-2", "2:-1"));
    samplesList.add(Arrays.asList("1:-1", "2:-1"));
    samplesList.add(Arrays.asList("1:-1", "2:-2"));
    samplesList.add(Arrays.asList("1:1", "2:1"));
    samplesList.add(Arrays.asList("1:1", "2:2"));
    samplesList.add(Arrays.asList("1:2", "2:1"));

    int[] labels = new int[] {0, 0, 0, 1, 1, 1};

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, options);

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});

    for (int i = 0, size = samplesList.size(); i < size; i++) {
        udtf.process(new Object[] {samplesList.get(i), labels[i]});
    }

    udtf.finalizeTraining();

    double cumLoss = udtf.getCumulativeLoss();
    println("Cumulative loss: " + cumLoss);
    double normalizedLoss = cumLoss / samplesList.size();
    Assert.assertTrue(
        "cumLoss: " + cumLoss + ", normalizedLoss: " + normalizedLoss + "\noptions: " + options,
        normalizedLoss < 0.5d);

    int numTests = 0;
    int numCorrect = 0;

    for (int i = 0, size = samplesList.size(); i < size; i++) {
        int label = labels[i];

        float score = udtf.predict(udtf.parseFeatures(samplesList.get(i)));
        int predicted = score > 0.f ? 1 : 0;

        println("Score: " + score + ", Predicted: " + predicted + ", Actual: " + label);

        if (predicted == label) {
            ++numCorrect;
        }
        ++numTests;
    }

    float accuracy = numCorrect / (float) numTests;
    println("Accuracy: " + accuracy);
    Assert.assertTrue(accuracy == 1.f);
}

Example 16

Source File: EthereumUDFTest.java From hadoopcryptoledger with Apache License 2.0

4 votes

@Test
 public void EthereumGetSendAddressUDFObjectInspector() throws HiveException, IOException, EthereumBlockReadException {
  // initialize object inspector
  EthereumGetSendAddressUDF egsaUDF = new EthereumGetSendAddressUDF();
	ObjectInspector[] arguments = new ObjectInspector[2];
	arguments[0] =  ObjectInspectorFactory.getReflectionObjectInspector(TestEthereumTransaction.class,ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
	arguments[1] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
	egsaUDF.initialize(arguments);	
	// load test data
  ClassLoader classLoader = getClass().getClassLoader();
	String fileName="eth1346406.bin";
	String fileNameBlock=classLoader.getResource(fileName).getFile();	
	File file = new File(fileNameBlock);
	boolean direct=false;
	FileInputStream fin = new FileInputStream(file);
	EthereumBlockReader ebr = null;
	try {
		ebr = new EthereumBlockReader(fin,this.DEFAULT_MAXSIZE_ETHEREUMBLOCK, this.DEFAULT_BUFFERSIZE,direct);
		EthereumBlock eblock = ebr.readBlock();
		List<EthereumTransaction> eTrans = eblock.getEthereumTransactions();
		// validate UDFs
		EthereumTransaction transOrig0 = eTrans.get(0);
		TestEthereumTransaction trans0 = new TestEthereumTransaction();
		trans0.set(transOrig0);
	      byte[] expectedSentAddress = new byte[] {(byte)0x39,(byte)0x42,(byte)0x4b,(byte)0xd2,(byte)0x8a,(byte)0x22,(byte)0x23,(byte)0xda,(byte)0x3e,(byte)0x14,(byte)0xbf,(byte)0x79,(byte)0x3c,(byte)0xf7,(byte)0xf8,(byte)0x20,(byte)0x8e,(byte)0xe9,(byte)0x98,(byte)0x0a};
	      assertArrayEquals(expectedSentAddress,((BytesWritable)egsaUDF.evaluate(new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(trans0),new GenericUDF.DeferredJavaObject(new IntWritable(1))})).copyBytes(),"Block 1346406 Transaction 1 send address is correctly calculated");
			EthereumTransaction transOrig1 = eTrans.get(1);
			TestEthereumTransaction trans1 = new TestEthereumTransaction();
			trans1.set(transOrig1);
	      expectedSentAddress = new byte[] {(byte)0x4b,(byte)0xb9,(byte)0x60,(byte)0x91,(byte)0xee,(byte)0x9d,(byte)0x80,(byte)0x2e,(byte)0xd0,(byte)0x39,(byte)0xc4,(byte)0xd1,(byte)0xa5,(byte)0xf6,(byte)0x21,(byte)0x6f,(byte)0x90,(byte)0xf8,(byte)0x1b,(byte)0x01};
	      assertArrayEquals(expectedSentAddress,((BytesWritable)egsaUDF.evaluate(new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(trans1),new GenericUDF.DeferredJavaObject(new IntWritable(1))})).copyBytes(),"Block 1346406 Transaction 2 send address is correctly calculated");
			EthereumTransaction transOrig2 = eTrans.get(2);
			TestEthereumTransaction trans2 = new TestEthereumTransaction();
			trans2.set(transOrig2);
	      expectedSentAddress = new byte[] {(byte)0x63,(byte)0xa9,(byte)0x97,(byte)0x5b,(byte)0xa3,(byte)0x1b,(byte)0x0b,(byte)0x96,(byte)0x26,(byte)0xb3,(byte)0x43,(byte)0x00,(byte)0xf7,(byte)0xf6,(byte)0x27,(byte)0x14,(byte)0x7d,(byte)0xf1,(byte)0xf5,(byte)0x26};
	      assertArrayEquals(expectedSentAddress,((BytesWritable)egsaUDF.evaluate(new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(trans2),new GenericUDF.DeferredJavaObject(new IntWritable(1))})).copyBytes(),"Block 1346406 Transaction 3 send address is correctly calculated");
			EthereumTransaction transOrig3 = eTrans.get(3);
			TestEthereumTransaction trans3 = new TestEthereumTransaction();
			trans3.set(transOrig3);
	      expectedSentAddress = new byte[] {(byte)0x63,(byte)0xa9,(byte)0x97,(byte)0x5b,(byte)0xa3,(byte)0x1b,(byte)0x0b,(byte)0x96,(byte)0x26,(byte)0xb3,(byte)0x43,(byte)0x00,(byte)0xf7,(byte)0xf6,(byte)0x27,(byte)0x14,(byte)0x7d,(byte)0xf1,(byte)0xf5,(byte)0x26};
	     assertArrayEquals(expectedSentAddress,((BytesWritable)egsaUDF.evaluate(new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(trans3),new GenericUDF.DeferredJavaObject(new IntWritable(1))})).copyBytes(),"Block 1346406 Transaction 4 send address is correctly calculated");
			EthereumTransaction transOrig4 = eTrans.get(4);
			TestEthereumTransaction trans4 = new TestEthereumTransaction();
			trans4.set(transOrig4);
	      expectedSentAddress = new byte[] {(byte)0x63,(byte)0xa9,(byte)0x97,(byte)0x5b,(byte)0xa3,(byte)0x1b,(byte)0x0b,(byte)0x96,(byte)0x26,(byte)0xb3,(byte)0x43,(byte)0x00,(byte)0xf7,(byte)0xf6,(byte)0x27,(byte)0x14,(byte)0x7d,(byte)0xf1,(byte)0xf5,(byte)0x26};
	      assertArrayEquals(expectedSentAddress,((BytesWritable)egsaUDF.evaluate(new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(trans4),new GenericUDF.DeferredJavaObject(new IntWritable(1))})).copyBytes(),"Block 1346406 Transaction 5 send address is correctly calculated");
			EthereumTransaction transOrig5 = eTrans.get(5);
			TestEthereumTransaction trans5 = new TestEthereumTransaction();
			trans5.set(transOrig5);
	      expectedSentAddress = new byte[] {(byte)0x63,(byte)0xa9,(byte)0x97,(byte)0x5b,(byte)0xa3,(byte)0x1b,(byte)0x0b,(byte)0x96,(byte)0x26,(byte)0xb3,(byte)0x43,(byte)0x00,(byte)0xf7,(byte)0xf6,(byte)0x27,(byte)0x14,(byte)0x7d,(byte)0xf1,(byte)0xf5,(byte)0x26};
	      assertArrayEquals(expectedSentAddress,((BytesWritable)egsaUDF.evaluate(new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(trans5),new GenericUDF.DeferredJavaObject(new IntWritable(1))})).copyBytes(),"Block 1346406 Transaction 6 send address is correctly calculated");
	      
}
	 finally {
		if (ebr!=null) {
			ebr.close();
		}
	}

 }

Example 17

Source File: GeneralRegressorUDTFTest.java From incubator-hivemall with Apache License 2.0

4 votes

@Test
public void testIntegerFeature() throws Exception {
    List<Integer> x = Arrays.asList(111, 222);
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    testFeature(x, featureOI, Integer.class, Integer.class);
}

Example 18

Source File: MatrixFactorizationSGDUDTFTest.java From incubator-hivemall with Apache License 2.0

4 votes

@Test
public void testIterationsCloseWithoutFile() throws HiveException {
    println("--------------------------\n testIterationsCloseWithoutFile()");
    OnlineMatrixFactorizationUDTF mf = new MatrixFactorizationSGDUDTF();

    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    int iters = 3;
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        new String("-factor 3 -iterations " + iters));
    ObjectInspector[] argOIs = new ObjectInspector[] {intOI, intOI, floatOI, param};
    MapredContext mrContext = MapredContextAccessor.create(true, null);
    mf.configure(mrContext);
    mf.initialize(argOIs);
    final MutableInt numCollected = new MutableInt(0);
    mf.setCollector(new Collector() {
        @Override
        public void collect(Object input) throws HiveException {
            numCollected.addValue(1);
        }
    });
    Assert.assertTrue(mf.rankInit == RankInitScheme.random);

    float[][] rating = {{5, 3, 0, 1}, {4, 0, 0, 1}, {1, 1, 0, 5}, {1, 0, 0, 4}, {0, 1, 5, 4}};
    Object[] args = new Object[3];

    final int num_iters = 100;
    int trainingExamples = 0;
    for (int iter = 0; iter < num_iters; iter++) {
        for (int row = 0; row < rating.length; row++) {
            for (int col = 0, size = rating[row].length; col < size; col++) {
                args[0] = row;
                args[1] = col;
                args[2] = (float) rating[row][col];
                mf.process(args);
                trainingExamples++;
            }
        }
    }
    mf.close();
    Assert.assertEquals(trainingExamples * iters, mf.count);
    Assert.assertEquals(5, numCollected.intValue());
}

Example 19

Source File: MatrixFactorizationSGDUDTFTest.java From incubator-hivemall with Apache License 2.0

4 votes

@Test
public void testFileBackedIterationsCloseNoConverge() throws HiveException {
    println("--------------------------\n testFileBackedIterationsCloseNoConverge()");
    OnlineMatrixFactorizationUDTF mf = new MatrixFactorizationSGDUDTF();

    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    int iters = 5;
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        new String("-disable_cv -factor 3 -iterations " + iters));
    ObjectInspector[] argOIs = new ObjectInspector[] {intOI, intOI, floatOI, param};
    MapredContext mrContext = MapredContextAccessor.create(true, null);
    mf.configure(mrContext);
    mf.initialize(argOIs);
    final MutableInt numCollected = new MutableInt(0);
    mf.setCollector(new Collector() {
        @Override
        public void collect(Object input) throws HiveException {
            numCollected.addValue(1);
        }
    });
    Assert.assertTrue(mf.rankInit == RankInitScheme.random);

    float[][] rating = {{5, 3, 0, 1}, {4, 0, 0, 1}, {1, 1, 0, 5}, {1, 0, 0, 4}, {0, 1, 5, 4}};
    Object[] args = new Object[3];

    final int num_iters = 500;
    int trainingExamples = 0;
    for (int iter = 0; iter < num_iters; iter++) {
        for (int row = 0; row < rating.length; row++) {
            for (int col = 0, size = rating[row].length; col < size; col++) {
                args[0] = row;
                args[1] = col;
                args[2] = (float) rating[row][col];
                mf.process(args);
                trainingExamples++;
            }
        }
    }

    File tmpFile = mf.fileIO.getFile();
    mf.close();
    Assert.assertEquals(trainingExamples * iters, mf.count);
    Assert.assertEquals(5, numCollected.intValue());
    Assert.assertFalse(tmpFile.exists());
}

Example 20

Source File: TestParquetHiveArrayInspector.java From parquet-mr with Apache License 2.0

4 votes

@Override
public void setUp() {
  inspector = new ParquetHiveArrayInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
}