Java Code Examples for org.apache.flink.types.Value

The following examples show how to use org.apache.flink.types.Value. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: SyncEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 
Example 2
Source Project: flink   Source File: RecordPairComparatorFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypePairComparator<Record, Record> createComparator12(
		TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos1, pos2, types1);
}
 
Example 3
Source Project: flink   Source File: AggregatorRegistry.java    License: Apache License 2.0 6 votes vote down vote up
public <T extends Value> void registerAggregationConvergenceCriterion(
		String name, Aggregator<T> aggregator, ConvergenceCriterion<T> convergenceCheck)
{
	if (name == null || aggregator == null || convergenceCheck == null) {
		throw new IllegalArgumentException("Name, aggregator, or convergence criterion must not be null");
	}
	
	Aggregator<?> genAgg = aggregator;
	
	Aggregator<?> previous = this.registry.get(name);
	if (previous != null && previous != genAgg) {
		throw new RuntimeException("An aggregator is already registered under the given name.");
	}
	
	this.registry.put(name, genAgg);
	this.convergenceCriterion = convergenceCheck;
	this.convergenceCriterionAggregatorName = name;
}
 
Example 4
Source Project: flink   Source File: ValueArrayFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 6
Source Project: flink   Source File: RecordPairComparatorFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypePairComparator<Record, Record> createComparator21(
	TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos2, pos1, types1);
}
 
Example 7
Source Project: flink   Source File: RecordComparator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copy constructor.
 * 
 * @param toCopy Comparator to copy.
 */
private RecordComparator(RecordComparator toCopy) {
	this.keyFields = toCopy.keyFields;
	this.keyHolders = new Value[toCopy.keyHolders.length];
	this.transientKeyHolders = new Value[toCopy.keyHolders.length];
	
	try {
		for (int i = 0; i < this.keyHolders.length; i++) {
			this.keyHolders[i] = toCopy.keyHolders[i].getClass().newInstance();
			this.transientKeyHolders[i] = toCopy.keyHolders[i].getClass().newInstance();
		}
	} catch (Exception ex) {
		// this should never happen, because the classes have been instantiated before. Report for debugging.
		throw new RuntimeException("Could not instantiate key classes when duplicating RecordComparator.", ex);
	}
	
	this.normalizedKeyLengths = toCopy.normalizedKeyLengths;
	this.numLeadingNormalizableKeys = toCopy.numLeadingNormalizableKeys;
	this.normalizableKeyPrefixLen = toCopy.normalizableKeyPrefixLen;
	this.ascending = toCopy.ascending;
	
	this.temp1 = new Record();
	this.temp2 = new Record();
}
 
Example 8
Source Project: flink   Source File: TypeExtractorTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testValueSupertypeException() {
	RichMapFunction<?, ?> function = new RichMapFunction<StringValue, Value>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Value map(StringValue value) throws Exception {
			return null;
		}
	};

	TypeInformation<?> ti =TypeExtractor.getMapReturnTypes(function, (TypeInformation)TypeInformation.of(new TypeHint<StringValue>(){}), "name", true);
	Assert.assertTrue(ti instanceof MissingTypeInfo);
	
	try {
		TypeExtractor.getMapReturnTypes(function, (TypeInformation)TypeInformation.of(new TypeHint<StringValue>(){}));
		Assert.fail("Expected an exception");
	}
	catch (InvalidTypesException e) {
		// expected
	}
}
 
Example 9
Source Project: flink   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();

	this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(32 * 1024 * 1024).build();
	this.ioManager = new IOManagerAsync();
}
 
Example 10
Source Project: Flink-CEPplus   Source File: ValueArrayFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example 11
Source Project: flink   Source File: ValueArrayFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example 12
Source Project: flink   Source File: RecordPairComparator.java    License: Apache License 2.0 6 votes vote down vote up
public RecordPairComparator(int[] keyFieldsReference, int[] keyFieldsCandidate, Class<? extends Value>[] keyTypes) {
	if (keyFieldsReference.length != keyFieldsCandidate.length || keyFieldsCandidate.length != keyTypes.length) {
		throw new IllegalArgumentException(
			"The arrays describing the key positions and types must be of the same length.");
	}
	this.keyFields1 = keyFieldsReference;
	this.keyFields2 = keyFieldsCandidate;
	
	// instantiate fields to extract keys into
	this.keyHolders1 = new Value[keyTypes.length];
	this.keyHolders2 = new Value[keyTypes.length];
	
	for (int i = 0; i < keyTypes.length; i++) {
		if (keyTypes[i] == null) {
			throw new NullPointerException("Key type " + i + " is null.");
		}
		this.keyHolders1[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
		this.keyHolders2[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
	}
}
 
Example 13
Source Project: flink   Source File: RecordComparatorFactory.java    License: Apache License 2.0 6 votes vote down vote up
public RecordComparatorFactory(int[] positions, Class<? extends Value>[] types, boolean[] sortDirections) {
	if (positions == null || types == null) {
		throw new NullPointerException();
	}
	if (positions.length != types.length) {
		throw new IllegalArgumentException();
	}
	
	this.positions = positions;
	this.types = types;
	
	if (sortDirections == null) {
		this.sortDirections = new boolean[positions.length];
		Arrays.fill(this.sortDirections, true);
	} else if (sortDirections.length != positions.length) {
		throw new IllegalArgumentException();
	} else {
		this.sortDirections = sortDirections;
	}
}
 
Example 14
protected IterationEventWithAggregators(Map<String, Aggregator<?>> aggregators) {
	int num = aggregators.size();
	if (num == 0) {
		this.aggNames = NO_STRINGS;
		this.aggregates = NO_VALUES;
	} else {
		this.aggNames = new String[num];
		this.aggregates = new Value[num];

		int i = 0;
		for (Map.Entry<String, Aggregator<?>> entry : aggregators.entrySet()) {
			this.aggNames[i] = entry.getKey();
			this.aggregates[i] = entry.getValue().getAggregate();
			i++;
		}
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 
Example 16
Source Project: Flink-CEPplus   Source File: RecordPairComparatorFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypePairComparator<Record, Record> createComparator21(
	TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos2, pos1, types1);
}
 
Example 17
Source Project: Flink-CEPplus   Source File: RecordPairComparator.java    License: Apache License 2.0 6 votes vote down vote up
public RecordPairComparator(int[] keyFieldsReference, int[] keyFieldsCandidate, Class<? extends Value>[] keyTypes) {
	if (keyFieldsReference.length != keyFieldsCandidate.length || keyFieldsCandidate.length != keyTypes.length) {
		throw new IllegalArgumentException(
			"The arrays describing the key positions and types must be of the same length.");
	}
	this.keyFields1 = keyFieldsReference;
	this.keyFields2 = keyFieldsCandidate;
	
	// instantiate fields to extract keys into
	this.keyHolders1 = new Value[keyTypes.length];
	this.keyHolders2 = new Value[keyTypes.length];
	
	for (int i = 0; i < keyTypes.length; i++) {
		if (keyTypes[i] == null) {
			throw new NullPointerException("Key type " + i + " is null.");
		}
		this.keyHolders1[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
		this.keyHolders2[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
	}
}
 
Example 18
Source Project: Flink-CEPplus   Source File: RecordComparatorFactory.java    License: Apache License 2.0 6 votes vote down vote up
public RecordComparatorFactory(int[] positions, Class<? extends Value>[] types, boolean[] sortDirections) {
	if (positions == null || types == null) {
		throw new NullPointerException();
	}
	if (positions.length != types.length) {
		throw new IllegalArgumentException();
	}
	
	this.positions = positions;
	this.types = types;
	
	if (sortDirections == null) {
		this.sortDirections = new boolean[positions.length];
		Arrays.fill(this.sortDirections, true);
	} else if (sortDirections.length != positions.length) {
		throw new IllegalArgumentException();
	} else {
		this.sortDirections = sortDirections;
	}
}
 
Example 19
Source Project: flink   Source File: SyncEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 
Example 20
Source Project: flink   Source File: ValueTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@PublicEvolving
public ValueTypeInfo(Class<T> type) {
	this.type = checkNotNull(type);

	checkArgument(
		Value.class.isAssignableFrom(type) || type.equals(Value.class),
		"ValueTypeInfo can only be used for subclasses of %s", Value.class.getName());
}
 
Example 21
Source Project: flink   Source File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
public CollectionExecutor(ExecutionConfig executionConfig) {
	this.executionConfig = executionConfig;
	
	this.intermediateResults = new HashMap<Operator<?>, List<?>>();
	this.accumulators = new HashMap<String, Accumulator<?,?>>();
	this.previousAggregates = new HashMap<String, Value>();
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.cachedFiles = new HashMap<String, Future<Path>>();
	this.userCodeClassLoader = Thread.currentThread().getContextClassLoader();
}
 
Example 22
Source Project: Flink-CEPplus   Source File: ValueTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@PublicEvolving
public ValueTypeInfo(Class<T> type) {
	this.type = checkNotNull(type);

	checkArgument(
		Value.class.isAssignableFrom(type) || type.equals(Value.class),
		"ValueTypeInfo can only be used for subclasses of %s", Value.class.getName());
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ValueTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@PublicEvolving
static <X extends Value> TypeInformation<X> getValueTypeInfo(Class<X> typeClass) {
	if (Value.class.isAssignableFrom(typeClass) && !typeClass.equals(Value.class)) {
		return new ValueTypeInfo<X>(typeClass);
	}
	else {
		throw new InvalidTypesException("The given class is no subclass of " + Value.class.getName());
	}
}
 
Example 24
Source Project: Flink-CEPplus   Source File: AggregatorRegistry.java    License: Apache License 2.0 5 votes vote down vote up
public Collection<AggregatorWithName<?>> getAllRegisteredAggregators() {
	ArrayList<AggregatorWithName<?>> list = new ArrayList<AggregatorWithName<?>>(this.registry.size());
	
	for (Map.Entry<String, Aggregator<?>> entry : this.registry.entrySet()) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> valAgg = (Aggregator<Value>) entry.getValue();
		list.add(new AggregatorWithName<>(entry.getKey(), valAgg));
	}
	return list;
}
 
Example 25
Source Project: flink   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void readWithEmptyField() {
	try {
		final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
		final FileInputSplit split = createTempFile(fileContent);
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, StringValue.class, StringValue.class);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new StringValue(), new StringValue(), new StringValue()};
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("abc", ((StringValue) values[0]).getValue());
		assertEquals("def", ((StringValue) values[1]).getValue());
		assertEquals("ghijk", ((StringValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("abc", ((StringValue) values[0]).getValue());
		assertEquals("", ((StringValue) values[1]).getValue());
		assertEquals("hhg", ((StringValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("", ((StringValue) values[0]).getValue());
		assertEquals("", ((StringValue) values[1]).getValue());
		assertEquals("", ((StringValue) values[2]).getValue());
		
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 26
Source Project: flink   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReadNoPosAllGzip() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
		final FileInputSplit split = createTempGzipFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);

		format.configure(parameters);
		format.open(split);

		Value[] values = createIntValues(5);

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		assertEquals(444, ((IntValue) values[3]).getValue());
		assertEquals(555, ((IntValue) values[4]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		assertEquals(999, ((IntValue) values[3]).getValue());
		assertEquals(000, ((IntValue) values[4]).getValue());

		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 27
Source Project: Flink-CEPplus   Source File: RichOutputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 
Example 28
Source Project: flink   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReadNoPosAllDeflate() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
		final FileInputSplit split = createTempDeflateFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);

		format.configure(parameters);
		format.open(split);

		Value[] values = createIntValues(5);

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		assertEquals(444, ((IntValue) values[3]).getValue());
		assertEquals(555, ((IntValue) values[4]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		assertEquals(999, ((IntValue) values[3]).getValue());
		assertEquals(000, ((IntValue) values[4]).getValue());

		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReadNoPosAllGzip() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
		final FileInputSplit split = createTempGzipFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);

		format.configure(parameters);
		format.open(split);

		Value[] values = createIntValues(5);

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		assertEquals(444, ((IntValue) values[3]).getValue());
		assertEquals(555, ((IntValue) values[4]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		assertEquals(999, ((IntValue) values[3]).getValue());
		assertEquals(000, ((IntValue) values[4]).getValue());

		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSparseParse() {
	try {
		final String fileContent =
				"111|222|333|444|555|666|777|888|999|000|\n"+
				"000|999|888|777|666|555|444|333|222|111|";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, null, null, IntValue.class, null, null, null, IntValue.class);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(3);
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(444, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(000, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		
		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		System.err.println(ex.getMessage());
		ex.printStackTrace();
		fail("Test erroneous");
	}
}