Java Code Examples for org.apache.spark.api.java.JavaRDD#fold()

public static MatrixBlock sumStable( JavaRDD<MatrixBlock> in )
{
	//stable sum of all blocks with correction block per function instance
	if( TREE_AGGREGATION ) {
		return in.treeReduce( 
				new SumSingleBlockFunction(true) );	
	}
	else { //DEFAULT
		//reduce-all aggregate via fold instead of reduce to allow 
		//for update in-place w/o deep copy of left-hand-side blocks
		return in.fold(
				new MatrixBlock(), 
				new SumSingleBlockFunction(false));
	}
}
 

/**
 * Single block aggregation over rdds with corrections for numerical stability.
 * 
 * @param in matrix as {@code JavaRDD<MatrixBlock>}
 * @param aop aggregate operator
 * @return matrix block
 */
public static MatrixBlock aggStable( JavaRDD<MatrixBlock> in, AggregateOperator aop )
{
	//stable aggregate of all blocks with correction block per function instance
	
	//reduce-all aggregate via fold instead of reduce to allow 
	//for update in-place w/o deep copy of left-hand-side blocks
	return in.fold(
			new MatrixBlock(),
			new AggregateSingleBlockFunction(aop) );
}
 

/**
 * Single block aggregation over rdds with corrections for numerical stability.
 *
 * @param in tensor as {@code JavaRDD<TensorBlock>}
 * @param aop aggregate operator
 * @return tensor block
 */
public static TensorBlock aggStableTensor(JavaRDD<TensorBlock> in, AggregateOperator aop )
{
	//stable aggregate of all blocks with correction block per function instance

	//reduce-all aggregate via fold instead of reduce to allow
	//for update in-place w/o deep copy of left-hand-side blocks
	return in.fold(
			new TensorBlock(),
			new AggregateSingleTensorBlockFunction(aop) );
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testReadDataFromIgnite() throws Exception {
    JavaSparkContext sc = new JavaSparkContext("local[*]", "test");

    try {
        JavaIgniteContext<String, Integer> ic = new JavaIgniteContext<>(sc, new IgniteConfigProvider());

        Ignite ignite = Ignition.ignite("grid-0");

        IgniteCache<String, Integer> cache = ignite.cache(ENTITY_CACHE_NAME);

        for (int i = 0; i < KEYS_CNT; i++)
            cache.put(String.valueOf(i), i);

        JavaRDD<Integer> values = ic.fromCache(ENTITY_CACHE_NAME).map(STR_INT_PAIR_TO_INT_F);

        int sum = values.fold(0, SUM_F);

        int expSum = (KEYS_CNT * KEYS_CNT + KEYS_CNT) / 2 - KEYS_CNT;

        assertEquals(expSum, sum);
    }
    finally {
        sc.stop();
    }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testReadDataFromIgnite() throws Exception {
    JavaSparkContext sc = createContext();

    JavaIgniteContext<String, Integer> ic = null;

    try {
        ic = new JavaIgniteContext<>(sc, new IgniteConfigProvider(), false);

        Ignite ignite = ic.ignite();

        IgniteCache<String, Integer> cache = ignite.cache(PARTITIONED_CACHE_NAME);

        for (int i = 0; i < KEYS_CNT; i++)
            cache.put(String.valueOf(i), i);

        JavaRDD<Integer> values = ic.fromCache(PARTITIONED_CACHE_NAME).map(STR_INT_PAIR_TO_INT_F);

        int sum = values.fold(0, SUM_F);

        int expSum = (KEYS_CNT * KEYS_CNT + KEYS_CNT) / 2 - KEYS_CNT;

        assertEquals(expSum, sum);
    }
    finally {
        if (ic != null)
            ic.close(true);

        sc.stop();
    }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testReadDataFromIgnite() throws Exception {
    JavaSparkContext sc = new JavaSparkContext("local[*]", "test");

    try {
        JavaIgniteContext<String, Integer> ic = new JavaIgniteContext<>(sc, new IgniteConfigProvider());

        Ignite ignite = Ignition.ignite("grid-0");

        IgniteCache<String, Integer> cache = ignite.cache(ENTITY_CACHE_NAME);

        for (int i = 0; i < KEYS_CNT; i++)
            cache.put(String.valueOf(i), i);

        JavaRDD<Integer> values = ic.fromCache(ENTITY_CACHE_NAME).map(STR_INT_PAIR_TO_INT_F);

        int sum = values.fold(0, SUM_F);

        int expSum = (KEYS_CNT * KEYS_CNT + KEYS_CNT) / 2 - KEYS_CNT;

        assertEquals(expSum, sum);
    }
    finally {
        sc.stop();
    }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testReadDataFromIgnite() throws Exception {
    JavaSparkContext sc = createContext();

    JavaIgniteContext<String, Integer> ic = null;

    try {
        ic = new JavaIgniteContext<>(sc, new IgniteConfigProvider(), false);

        Ignite ignite = ic.ignite();

        IgniteCache<String, Integer> cache = ignite.cache(PARTITIONED_CACHE_NAME);

        for (int i = 0; i < KEYS_CNT; i++)
            cache.put(String.valueOf(i), i);

        JavaRDD<Integer> values = ic.fromCache(PARTITIONED_CACHE_NAME).map(STR_INT_PAIR_TO_INT_F);

        int sum = values.fold(0, SUM_F);

        int expSum = (KEYS_CNT * KEYS_CNT + KEYS_CNT) / 2 - KEYS_CNT;

        assertEquals(expSum, sum);
    }
    finally {
        if (ic != null)
            ic.close(true);

        sc.stop();
    }
}
 

public static MatrixBlock sumStable( JavaRDD<MatrixBlock> in )
{
	//stable sum of all blocks with correction block per function instance
	if( TREE_AGGREGATION ) {
		return in.treeReduce( 
				new SumSingleBlockFunction(true) );	
	}
	else { //DEFAULT
		//reduce-all aggregate via fold instead of reduce to allow 
		//for update in-place w/o deep copy of left-hand-side blocks
		return in.fold(
				new MatrixBlock(), 
				new SumSingleBlockFunction(false));
	}
}
 

/**
 * Single block aggregation over rdds with corrections for numerical stability.
 * 
 * @param in matrix as {@code JavaRDD<MatrixBlock>}
 * @param aop aggregate operator
 * @return matrix block
 */
public static MatrixBlock aggStable( JavaRDD<MatrixBlock> in, AggregateOperator aop )
{
	//stable aggregate of all blocks with correction block per function instance
	
	//reduce-all aggregate via fold instead of reduce to allow 
	//for update in-place w/o deep copy of left-hand-side blocks
	return in.fold(
			new MatrixBlock(),
			new AggregateSingleBlockFunction(aop) );
}
 

/**
 * Single block aggregation over rdds with corrections for numerical stability.
 *
 * @param in tensor as {@code JavaRDD<TensorBlock>}
 * @param aop aggregate operator
 * @return tensor block
 */
public static TensorBlock aggStableTensor(JavaRDD<TensorBlock> in, AggregateOperator aop )
{
	//stable aggregate of all blocks with correction block per function instance

	//reduce-all aggregate via fold instead of reduce to allow
	//for update in-place w/o deep copy of left-hand-side blocks
	return in.fold(
			new TensorBlock(),
			new AggregateSingleTensorBlockFunction(aop) );
}
 

@Test
public void testAdd() {
    List<INDArray> list = new ArrayList<>();
    for (int i = 0; i < 5; i++)
        list.add(Nd4j.ones(5));
    JavaRDD<INDArray> rdd = sc.parallelize(list);
    INDArray sum = rdd.fold(Nd4j.zeros(5), new Add());
    assertEquals(25, sum.sum(Integer.MAX_VALUE).getDouble(0), 1e-1);
}