org.apache.flink.types.Value Java Examples

The following examples show how to use org.apache.flink.types.Value. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TypeExtractorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testValueSupertypeException() {
	RichMapFunction<?, ?> function = new RichMapFunction<StringValue, Value>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Value map(StringValue value) throws Exception {
			return null;
		}
	};

	TypeInformation<?> ti =TypeExtractor.getMapReturnTypes(function, (TypeInformation)TypeInformation.of(new TypeHint<StringValue>(){}), "name", true);
	Assert.assertTrue(ti instanceof MissingTypeInfo);
	
	try {
		TypeExtractor.getMapReturnTypes(function, (TypeInformation)TypeInformation.of(new TypeHint<StringValue>(){}));
		Assert.fail("Expected an exception");
	}
	catch (InvalidTypesException e) {
		// expected
	}
}
 
Example #2
Source File: RecordPairComparator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public RecordPairComparator(int[] keyFieldsReference, int[] keyFieldsCandidate, Class<? extends Value>[] keyTypes) {
	if (keyFieldsReference.length != keyFieldsCandidate.length || keyFieldsCandidate.length != keyTypes.length) {
		throw new IllegalArgumentException(
			"The arrays describing the key positions and types must be of the same length.");
	}
	this.keyFields1 = keyFieldsReference;
	this.keyFields2 = keyFieldsCandidate;
	
	// instantiate fields to extract keys into
	this.keyHolders1 = new Value[keyTypes.length];
	this.keyHolders2 = new Value[keyTypes.length];
	
	for (int i = 0; i < keyTypes.length; i++) {
		if (keyTypes[i] == null) {
			throw new NullPointerException("Key type " + i + " is null.");
		}
		this.keyHolders1[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
		this.keyHolders2[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
	}
}
 
Example #3
Source File: AggregatorRegistry.java    From flink with Apache License 2.0 6 votes vote down vote up
public <T extends Value> void registerAggregationConvergenceCriterion(
		String name, Aggregator<T> aggregator, ConvergenceCriterion<T> convergenceCheck)
{
	if (name == null || aggregator == null || convergenceCheck == null) {
		throw new IllegalArgumentException("Name, aggregator, or convergence criterion must not be null");
	}
	
	Aggregator<?> genAgg = aggregator;
	
	Aggregator<?> previous = this.registry.get(name);
	if (previous != null && previous != genAgg) {
		throw new RuntimeException("An aggregator is already registered under the given name.");
	}
	
	this.registry.put(name, genAgg);
	this.convergenceCriterion = convergenceCheck;
	this.convergenceCriterionAggregatorName = name;
}
 
Example #4
Source File: RecordPairComparatorFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypePairComparator<Record, Record> createComparator12(
		TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos1, pos2, types1);
}
 
Example #5
Source File: RecordComparatorFactory.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public RecordComparatorFactory(int[] positions, Class<? extends Value>[] types, boolean[] sortDirections) {
	if (positions == null || types == null) {
		throw new NullPointerException();
	}
	if (positions.length != types.length) {
		throw new IllegalArgumentException();
	}
	
	this.positions = positions;
	this.types = types;
	
	if (sortDirections == null) {
		this.sortDirections = new boolean[positions.length];
		Arrays.fill(this.sortDirections, true);
	} else if (sortDirections.length != positions.length) {
		throw new IllegalArgumentException();
	} else {
		this.sortDirections = sortDirections;
	}
}
 
Example #6
Source File: SyncEventHandler.java    From flink with Apache License 2.0 6 votes vote down vote up
private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 
Example #7
Source File: SyncEventHandler.java    From flink with Apache License 2.0 6 votes vote down vote up
private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 
Example #8
Source File: RecordPairComparatorFactory.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public TypePairComparator<Record, Record> createComparator21(
	TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos2, pos1, types1);
}
 
Example #9
Source File: ValueArrayFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example #10
Source File: IterationEventWithAggregators.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
protected IterationEventWithAggregators(Map<String, Aggregator<?>> aggregators) {
	int num = aggregators.size();
	if (num == 0) {
		this.aggNames = NO_STRINGS;
		this.aggregates = NO_VALUES;
	} else {
		this.aggNames = new String[num];
		this.aggregates = new Value[num];

		int i = 0;
		for (Map.Entry<String, Aggregator<?>> entry : aggregators.entrySet()) {
			this.aggNames[i] = entry.getKey();
			this.aggregates[i] = entry.getValue().getAggregate();
			i++;
		}
	}
}
 
Example #11
Source File: GenericCsvInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example #12
Source File: RecordPairComparator.java    From flink with Apache License 2.0 6 votes vote down vote up
public RecordPairComparator(int[] keyFieldsReference, int[] keyFieldsCandidate, Class<? extends Value>[] keyTypes) {
	if (keyFieldsReference.length != keyFieldsCandidate.length || keyFieldsCandidate.length != keyTypes.length) {
		throw new IllegalArgumentException(
			"The arrays describing the key positions and types must be of the same length.");
	}
	this.keyFields1 = keyFieldsReference;
	this.keyFields2 = keyFieldsCandidate;
	
	// instantiate fields to extract keys into
	this.keyHolders1 = new Value[keyTypes.length];
	this.keyHolders2 = new Value[keyTypes.length];
	
	for (int i = 0; i < keyTypes.length; i++) {
		if (keyTypes[i] == null) {
			throw new NullPointerException("Key type " + i + " is null.");
		}
		this.keyHolders1[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
		this.keyHolders2[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
	}
}
 
Example #13
Source File: RecordComparatorFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
public RecordComparatorFactory(int[] positions, Class<? extends Value>[] types, boolean[] sortDirections) {
	if (positions == null || types == null) {
		throw new NullPointerException();
	}
	if (positions.length != types.length) {
		throw new IllegalArgumentException();
	}
	
	this.positions = positions;
	this.types = types;
	
	if (sortDirections == null) {
		this.sortDirections = new boolean[positions.length];
		Arrays.fill(this.sortDirections, true);
	} else if (sortDirections.length != positions.length) {
		throw new IllegalArgumentException();
	} else {
		this.sortDirections = sortDirections;
	}
}
 
Example #14
Source File: ValueArrayFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example #15
Source File: ValueArrayFactory.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example #16
Source File: RecordPairComparatorFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypePairComparator<Record, Record> createComparator21(
	TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos2, pos1, types1);
}
 
Example #17
Source File: HashTableITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 
Example #18
Source File: RecordComparator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Copy constructor.
 * 
 * @param toCopy Comparator to copy.
 */
private RecordComparator(RecordComparator toCopy) {
	this.keyFields = toCopy.keyFields;
	this.keyHolders = new Value[toCopy.keyHolders.length];
	this.transientKeyHolders = new Value[toCopy.keyHolders.length];
	
	try {
		for (int i = 0; i < this.keyHolders.length; i++) {
			this.keyHolders[i] = toCopy.keyHolders[i].getClass().newInstance();
			this.transientKeyHolders[i] = toCopy.keyHolders[i].getClass().newInstance();
		}
	} catch (Exception ex) {
		// this should never happen, because the classes have been instantiated before. Report for debugging.
		throw new RuntimeException("Could not instantiate key classes when duplicating RecordComparator.", ex);
	}
	
	this.normalizedKeyLengths = toCopy.normalizedKeyLengths;
	this.numLeadingNormalizableKeys = toCopy.numLeadingNormalizableKeys;
	this.normalizableKeyPrefixLen = toCopy.normalizableKeyPrefixLen;
	this.ascending = toCopy.ascending;
	
	this.temp1 = new Record();
	this.temp2 = new Record();
}
 
Example #19
Source File: HashTableITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();

	this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(32 * 1024 * 1024).build();
	this.ioManager = new IOManagerAsync();
}
 
Example #20
Source File: RecordComparator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public boolean equalToReference(Record candidate) {
	for (int i = 0; i < this.keyFields.length; i++) {
		final Value k = candidate.getField(this.keyFields[i], this.transientKeyHolders[i]);
		if (k == null) {
			throw new NullKeyFieldException(this.keyFields[i]);
		} else if (!k.equals(this.keyHolders[i])) {
			return false;
		}
	}
	return true;
}
 
Example #21
Source File: GenericCsvInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void readWithHeaderLineAndInvalidIntermediate() {
	try {
		final String fileContent = "colname-1|colname-2|some name 3|column four|\n" +
								"123|abc|456|def|\n"+
								"colname-1|colname-2|some name 3|column four|\n" +  // repeated header in the middle
								"987|xyz|654|pqr|\n";
		
		final FileInputSplit split = createTempFile(fileContent);
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, StringValue.class, IntValue.class, StringValue.class);
		format.setSkipFirstLineAsHeader(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new IntValue(), new StringValue(), new IntValue(), new StringValue()};
		
		// first line is skipped as header
		assertNotNull(format.nextRecord(values));   //  first row (= second line)
		
		try {
			format.nextRecord(values);
			fail("Format accepted invalid line.");
		}
		catch (ParseException e) {
			// as we expected
		}
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example #22
Source File: AggregatorRegistry.java    From flink with Apache License 2.0 5 votes vote down vote up
public Collection<AggregatorWithName<?>> getAllRegisteredAggregators() {
	ArrayList<AggregatorWithName<?>> list = new ArrayList<AggregatorWithName<?>>(this.registry.size());
	
	for (Map.Entry<String, Aggregator<?>> entry : this.registry.entrySet()) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> valAgg = (Aggregator<Value>) entry.getValue();
		list.add(new AggregatorWithName<>(entry.getKey(), valAgg));
	}
	return list;
}
 
Example #23
Source File: ValueComparator.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public int getNormalizeKeyLen() {
	if (reference == null) {
		reference = InstantiationUtil.instantiate(type, Value.class);
	}
	
	NormalizableKey<?> key = (NormalizableKey<?>) reference;
	return key.getMaxNormalizedKeyLen();
}
 
Example #24
Source File: RuntimeAggregatorRegistry.java    From flink with Apache License 2.0 5 votes vote down vote up
public void updateGlobalAggregatesAndReset(String[] names, Value[] aggregates) {
	if (names == null || aggregates == null || names.length != aggregates.length) {
		throw new IllegalArgumentException();
	}

	// add global aggregates
	for (int i = 0; i < names.length; i++) {
		this.previousGlobalAggregate.put(names[i], aggregates[i]);
	}

	// reset all aggregators
	for (Aggregator<?> agg : this.aggregators.values()) {
		agg.reset();
	}
}
 
Example #25
Source File: ValueTypeInfo.java    From flink with Apache License 2.0 5 votes vote down vote up
@PublicEvolving
static <X extends Value> TypeInformation<X> getValueTypeInfo(Class<X> typeClass) {
	if (Value.class.isAssignableFrom(typeClass) && !typeClass.equals(Value.class)) {
		return new ValueTypeInfo<X>(typeClass);
	}
	else {
		throw new InvalidTypesException("The given class is no subclass of " + Value.class.getName());
	}
}
 
Example #26
Source File: GenericCsvInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testReadNoPosFirstN() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555|\n666|777|888|999|000|";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(2);

		// if this would parse all, we would get an index out of bounds exception
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());

		
		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
	
}
 
Example #27
Source File: GenericCsvInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void readWithHeaderLine() {
	try {
		final String fileContent = "colname-1|colname-2|some name 3|column four|\n" +
								"123|abc|456|def|\n"+
								"987|xyz|654|pqr|\n";
		
		final FileInputSplit split = createTempFile(fileContent);
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, StringValue.class, IntValue.class, StringValue.class);
		format.setSkipFirstLineAsHeader(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new IntValue(), new StringValue(), new IntValue(), new StringValue()};
		
		// first line is skipped as header
		assertNotNull(format.nextRecord(values));   //  first row (= second line)
		assertNotNull(format.nextRecord(values));   // second row (= third line) 
		assertNull(format.nextRecord(values));  // exhausted
		assertTrue(format.reachedEnd());         // exhausted
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example #28
Source File: RecordComparatorFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void readParametersFromConfig(Configuration config, ClassLoader cl) throws ClassNotFoundException {
	// figure out how many key fields there are
	final int numKeyFields = config.getInteger(NUM_KEYS, -1);
	if (numKeyFields < 0) {
		throw new IllegalConfigurationException("The number of keys for the comparator is invalid: " + numKeyFields);
	}
	
	final int[] positions = new int[numKeyFields];
	final Class<? extends Value>[] types = new Class[numKeyFields];
	final boolean[] direction = new boolean[numKeyFields];
	
	// read the individual key positions and types
	for (int i = 0; i < numKeyFields; i++) {
		// next key position
		final int p = config.getInteger(KEY_POS_PREFIX + i, -1);
		if (p >= 0) {
			positions[i] = p;
		} else {
			throw new IllegalConfigurationException("Contained invalid position for key no positions for keys.");
		}
		
		// next key type
		final String name = config.getString(KEY_CLASS_PREFIX + i, null);
		if (name != null) {
			types[i] = (Class<? extends Value>) Class.forName(name, true, cl).asSubclass(Value.class);
		} else {
			throw new IllegalConfigurationException("The key type (" + i +
				") for the comparator is null"); 
		}
		
		// next key sort direction
		direction[i] = config.getBoolean(KEY_SORT_DIRECTION_PREFIX + i, true);
	}
	
	this.positions = positions;
	this.types = types;
	this.sortDirections = direction;
}
 
Example #29
Source File: RecordComparator.java    From flink with Apache License 2.0 5 votes vote down vote up
public final Value[] getKeysAsCopy(Record record) {
	try {
		final Value[] keys = new Value[this.keyFields.length];
		for (int i = 0; i < keys.length; i++) {
			keys[i] = this.keyHolders[i].getClass().newInstance();
		}
		if(!record.getFieldsInto(this.keyFields, keys)) {
			throw new RuntimeException("Could not extract keys from record.");
		}
		return keys;
	} catch (Exception ex) {
		// this should never happen, because the classes have been instantiated before. Report for debugging.
		throw new RuntimeException("Could not instantiate key classes when duplicating RecordComparator.", ex);
	}
}
 
Example #30
Source File: RecordComparator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public final Value[] getKeysAsCopy(Record record) {
	try {
		final Value[] keys = new Value[this.keyFields.length];
		for (int i = 0; i < keys.length; i++) {
			keys[i] = this.keyHolders[i].getClass().newInstance();
		}
		if(!record.getFieldsInto(this.keyFields, keys)) {
			throw new RuntimeException("Could not extract keys from record.");
		}
		return keys;
	} catch (Exception ex) {
		// this should never happen, because the classes have been instantiated before. Report for debugging.
		throw new RuntimeException("Could not instantiate key classes when duplicating RecordComparator.", ex);
	}
}