org.apache.flink.util.XORShiftRandom Java Examples

/**
 * Create a poisson sampler which can sample elements with replacement.
 *
 * @param fraction The expected count of each element.
 */
public PoissonSampler(double fraction) {
	Preconditions.checkArgument(fraction >= 0, "fraction should be non-negative.");
	this.fraction = fraction;
	if (this.fraction > 0) {
		this.poissonDistribution = new PoissonDistribution(fraction);
	}
	this.random = new XORShiftRandom();
}
 

/**
 * Create a poisson sampler which can sample elements with replacement.
 *
 * @param fraction The expected count of each element.
 * @param seed     Random number generator seed for internal PoissonDistribution.
 */
public PoissonSampler(double fraction, long seed) {
	Preconditions.checkArgument(fraction >= 0, "fraction should be positive.");
	this.fraction = fraction;
	if (this.fraction > 0) {
		this.poissonDistribution = new PoissonDistribution(fraction);
		this.poissonDistribution.reseedRandomGenerator(seed);
	}
	this.random = new XORShiftRandom(seed);
}
 

/**
 * Create a poisson sampler which can sample elements with replacement.
 *
 * @param fraction The expected count of each element.
 */
public PoissonSampler(double fraction) {
	Preconditions.checkArgument(fraction >= 0, "fraction should be non-negative.");
	this.fraction = fraction;
	if (this.fraction > 0) {
		this.poissonDistribution = new PoissonDistribution(fraction);
	}
	this.random = new XORShiftRandom();
}
 

/**
 * Create a poisson sampler which can sample elements with replacement.
 *
 * @param fraction The expected count of each element.
 * @param seed     Random number generator seed for internal PoissonDistribution.
 */
public PoissonSampler(double fraction, long seed) {
	Preconditions.checkArgument(fraction >= 0, "fraction should be positive.");
	this.fraction = fraction;
	if (this.fraction > 0) {
		this.poissonDistribution = new PoissonDistribution(fraction);
		this.poissonDistribution.reseedRandomGenerator(seed);
	}
	this.random = new XORShiftRandom(seed);
}
 

@Override
protected TaggedElement[] getTestData() {
    XORShiftRandom random = new XORShiftRandom();

    TaggedElement[] data = new TaggedElement[100];
    for (int i = 0; i < data.length; i++) {
        final int tag = random.nextInt(3);
        final Object element;
        switch (tag) {
            case 0: {
                element = random.nextLong();
                break;
            }
            case 1: {
                byte[] bytes = new byte[random.nextInt(256)];
                random.nextBytes(bytes);
                element = new String(bytes);
                break;
            }
            case 2: {
                element = random.nextBoolean();
                break;
            }
            default: {
                element = null;
            }
        }
        data[i] = new TaggedElement(tag, element);
    }
    return data;
}
 

/**
 * Create a poisson sampler which can sample elements with replacement.
 *
 * @param fraction The expected count of each element.
 * @param seed     Random number generator seed for internal PoissonDistribution.
 */
public PoissonSampler(double fraction, long seed) {
	Preconditions.checkArgument(fraction >= 0, "fraction should be positive.");
	this.fraction = fraction;
	if (this.fraction > 0) {
		this.poissonDistribution = new PoissonDistribution(fraction);
		this.poissonDistribution.reseedRandomGenerator(seed);
	}
	this.random = new XORShiftRandom(seed);
}
 

/**
 * Create a poisson sampler which can sample elements with replacement.
 *
 * @param fraction The expected count of each element.
 */
public PoissonSampler(double fraction) {
	Preconditions.checkArgument(fraction >= 0, "fraction should be non-negative.");
	this.fraction = fraction;
	if (this.fraction > 0) {
		this.poissonDistribution = new PoissonDistribution(fraction);
	}
	this.random = new XORShiftRandom();
}
 

/**
 * Tests broadcasting events when records have been emitted. The emitted
 * records cover all three {@link SerializationResult} types.
 */
@Test
public void testBroadcastEventMixedRecords() throws Exception {
	Random rand = new XORShiftRandom();
	int numberOfChannels = 4;
	int bufferSize = 32;
	int lenBytes = 4; // serialized length

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = new RecordWriter<>(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// Emit records on some channels first (requesting buffers), then
	// broadcast the event. The record buffers should be emitted first, then
	// the event. After the event, no new buffer should be requested.

	// (i) Smaller than the buffer size (single buffer request => 1)
	byte[] bytes = new byte[bufferSize / 2];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (ii) Larger than the buffer size (two buffer requests => 1 + 2)
	bytes = new byte[bufferSize + 1];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iii) Exactly the buffer size (single buffer request => 1 + 2 + 1)
	bytes = new byte[bufferSize - lenBytes];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iv) Nothing on the 4th channel (no buffer request => 1 + 2 + 1 + 0 = 4)

	// (v) Broadcast the event
	writer.broadcastEvent(barrier);

	assertEquals(4, bufferProvider.getNumberOfCreatedBuffers());

	BufferOrEvent boe;
	assertEquals(2, queues[0].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[0].remove(), 0).isBuffer());
	assertEquals(3, queues[1].size()); // 2 buffers + 1 event
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertEquals(2, queues[2].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[2].remove(), 2).isBuffer());
	assertEquals(1, queues[3].size()); // 0 buffers + 1 event

	// every queue's last element should be the event
	for (int i = 0; i < numberOfChannels; i++) {
		boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
	}
}
 

/**
 * Tests broadcasting events when records have been emitted. The emitted
 * records cover all three {@link SerializationResult} types.
 */
@Test
public void testBroadcastEventMixedRecords() throws Exception {
	Random rand = new XORShiftRandom();
	int numberOfChannels = 4;
	int bufferSize = 32;
	int lenBytes = 4; // serialized length

	@SuppressWarnings("unchecked")
	Queue<BufferConsumer>[] queues = new Queue[numberOfChannels];
	for (int i = 0; i < numberOfChannels; i++) {
		queues[i] = new ArrayDeque<>();
	}

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize);

	ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider);
	RecordWriter<ByteArrayIO> writer = new RecordWriterBuilder().build(partitionWriter);
	CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation());

	// Emit records on some channels first (requesting buffers), then
	// broadcast the event. The record buffers should be emitted first, then
	// the event. After the event, no new buffer should be requested.

	// (i) Smaller than the buffer size (single buffer request => 1)
	byte[] bytes = new byte[bufferSize / 2];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (ii) Larger than the buffer size (two buffer requests => 1 + 2)
	bytes = new byte[bufferSize + 1];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iii) Exactly the buffer size (single buffer request => 1 + 2 + 1)
	bytes = new byte[bufferSize - lenBytes];
	rand.nextBytes(bytes);

	writer.emit(new ByteArrayIO(bytes));

	// (iv) Nothing on the 4th channel (no buffer request => 1 + 2 + 1 + 0 = 4)

	// (v) Broadcast the event
	writer.broadcastEvent(barrier);

	assertEquals(4, bufferProvider.getNumberOfCreatedBuffers());

	BufferOrEvent boe;
	assertEquals(2, queues[0].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[0].remove(), 0).isBuffer());
	assertEquals(3, queues[1].size()); // 2 buffers + 1 event
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer());
	assertEquals(2, queues[2].size()); // 1 buffer + 1 event
	assertTrue(parseBuffer(queues[2].remove(), 2).isBuffer());
	assertEquals(1, queues[3].size()); // 0 buffers + 1 event

	// every queue's last element should be the event
	for (int i = 0; i < numberOfChannels; i++) {
		boe = parseBuffer(queues[i].remove(), i);
		assertTrue(boe.isEvent());
		assertEquals(barrier, boe.getEvent());
	}
}
 

/**
 * Create a Bernoulli sampler with sample fraction and random number generator seed.
 *
 * @param fraction Sample fraction, aka the Bernoulli sampler possibility.
 * @param seed     Random number generator seed.
 */
public BernoulliSampler(double fraction, long seed) {
	this(fraction, new XORShiftRandom(seed));
}
 

/**
 * Create a sampler with fixed sample size and default random number generator.
 *
 * @param numSamples Number of selected elements, must be non-negative.
 */
public ReservoirSamplerWithReplacement(int numSamples) {
	this(numSamples, new XORShiftRandom());
}
 

/**
 * Create a Bernoulli sampler with sample fraction and default random number generator.
 *
 * @param fraction Sample fraction, aka the Bernoulli sampler possibility.
 */
public BernoulliSampler(double fraction) {
	this(fraction, new XORShiftRandom());
}
 

/**
 * Create a new sampler with reservoir size and the seed for random number generator.
 *
 * @param numSamples Maximum number of samples to retain in reservoir, must be non-negative.
 * @param seed       Random number generator seed.
 */
public ReservoirSamplerWithoutReplacement(int numSamples, long seed) {

	this(numSamples, new XORShiftRandom(seed));
}
 

/**
 * Create a new sampler with reservoir size and a default random number generator.
 *
 * @param numSamples Maximum number of samples to retain in reservoir, must be non-negative.
 */
public ReservoirSamplerWithoutReplacement(int numSamples) {
	this(numSamples, new XORShiftRandom());
}
 

/**
 * Create a sampler with fixed sample size and random number generator seed.
 *
 * @param numSamples Number of selected elements, must be non-negative.
 * @param seed       Random number generator seed
 */
public ReservoirSamplerWithReplacement(int numSamples, long seed) {
	this(numSamples, new XORShiftRandom(seed));
}
 

/**
 * Create a sampler with fixed sample size and default random number generator.
 *
 * @param numSamples Number of selected elements, must be non-negative.
 */
public ReservoirSamplerWithReplacement(int numSamples) {
	this(numSamples, new XORShiftRandom());
}
 

/**
 * Create a Bernoulli sampler with sample fraction and random number generator seed.
 *
 * @param fraction Sample fraction, aka the Bernoulli sampler possibility.
 * @param seed     Random number generator seed.
 */
public BernoulliSampler(double fraction, long seed) {
	this(fraction, new XORShiftRandom(seed));
}
 

/**
 * Create a Bernoulli sampler with sample fraction and default random number generator.
 *
 * @param fraction Sample fraction, aka the Bernoulli sampler possibility.
 */
public BernoulliSampler(double fraction) {
	this(fraction, new XORShiftRandom());
}
 

/**
 * Create a new sampler with reservoir size and the seed for random number generator.
 *
 * @param numSamples Maximum number of samples to retain in reservoir, must be non-negative.
 * @param seed       Random number generator seed.
 */
public ReservoirSamplerWithoutReplacement(int numSamples, long seed) {

	this(numSamples, new XORShiftRandom(seed));
}
 

/**
 * Create a new sampler with reservoir size and a default random number generator.
 *
 * @param numSamples Maximum number of samples to retain in reservoir, must be non-negative.
 */
public ReservoirSamplerWithoutReplacement(int numSamples) {
	this(numSamples, new XORShiftRandom());
}
 

/**
 * Create a sampler with fixed sample size and random number generator seed.
 *
 * @param numSamples Number of selected elements, must be non-negative.
 * @param seed       Random number generator seed
 */
public ReservoirSamplerWithReplacement(int numSamples, long seed) {
	this(numSamples, new XORShiftRandom(seed));
}
 

/**
 * Create a sampler with fixed sample size and default random number generator.
 *
 * @param numSamples Number of selected elements, must be non-negative.
 */
public ReservoirSamplerWithReplacement(int numSamples) {
	this(numSamples, new XORShiftRandom());
}
 

/**
 * Create a Bernoulli sampler with sample fraction and random number generator seed.
 *
 * @param fraction Sample fraction, aka the Bernoulli sampler possibility.
 * @param seed     Random number generator seed.
 */
public BernoulliSampler(double fraction, long seed) {
	this(fraction, new XORShiftRandom(seed));
}
 

/**
 * Create a Bernoulli sampler with sample fraction and default random number generator.
 *
 * @param fraction Sample fraction, aka the Bernoulli sampler possibility.
 */
public BernoulliSampler(double fraction) {
	this(fraction, new XORShiftRandom());
}
 

/**
 * Create a new sampler with reservoir size and the seed for random number generator.
 *
 * @param numSamples Maximum number of samples to retain in reservoir, must be non-negative.
 * @param seed       Random number generator seed.
 */
public ReservoirSamplerWithoutReplacement(int numSamples, long seed) {

	this(numSamples, new XORShiftRandom(seed));
}
 

/**
 * Create a new sampler with reservoir size and a default random number generator.
 *
 * @param numSamples Maximum number of samples to retain in reservoir, must be non-negative.
 */
public ReservoirSamplerWithoutReplacement(int numSamples) {
	this(numSamples, new XORShiftRandom());
}
 

/**
 * Create a sampler with fixed sample size and random number generator seed.
 *
 * @param numSamples Number of selected elements, must be non-negative.
 * @param seed       Random number generator seed
 */
public ReservoirSamplerWithReplacement(int numSamples, long seed) {
	this(numSamples, new XORShiftRandom(seed));
}