Java Code Examples for org.apache.flink.types.StringValue

The following examples show how to use org.apache.flink.types.StringValue. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNull(gFields);
	assertNull(order);
	assertNull(filtered.getUniqueFields());
}
 
Example 2
Source Project: flink   Source File: StringValueArray.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String toString() {
	StringBuilder sb = new StringBuilder("[");
	String separator = "";

	for (StringValue sv : this) {
		sb
			.append(sv.getValue())
			.append(separator);
		separator = ",";
	}

	sb.append("]");

	return sb.toString();
}
 
Example 3
Source Project: flink   Source File: AggregateITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFullAggregateOfMutableValueTypes() throws Exception {
	/*
	 * Full Aggregate of mutable value types
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<IntValue, LongValue, StringValue>> ds = ValueCollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple2<IntValue, LongValue>> aggregateDs = ds
			.aggregate(Aggregations.SUM, 0)
			.and(Aggregations.MAX, 1)
			.project(0, 1);

	List<Tuple2<IntValue, LongValue>> result = aggregateDs.collect();

	String expected = "231,6\n";

	compareResultAsTuples(result, expected);
}
 
Example 4
Source Project: flink   Source File: StringValueArray.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean addAll(ValueArray<StringValue> other) {
	StringValueArray source = (StringValueArray) other;

	int sourceSize = source.position;
	int newPosition = position + sourceSize;

	if (newPosition > data.length) {
		if (isBounded) {
			return false;
		} else {
			ensureCapacity(newPosition);
		}
	}

	System.arraycopy(source.data, 0, data, position, sourceSize);
	length += source.length;
 	    position = newPosition;

	return true;
}
 
Example 5
Source Project: flink   Source File: StringValueUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Gets the next token from the string. If another token is available, the token is stored
 * in the given target StringValue object.
 *
 * @param target The StringValue object to store the next token in.
 * @return True, if there was another token, false if not.
 */
public boolean next(StringValue target) {
	final char[] data = this.toTokenize.getCharArray();
	final int limit = this.limit;
	int pos = this.pos;

	// skip the delimiter
	for (; pos < limit && Character.isWhitespace(data[pos]); pos++) {
	}

	if (pos >= limit) {
		this.pos = pos;
		return false;
	}

	final int start = pos;
	for (; pos < limit && !Character.isWhitespace(data[pos]); pos++) {
	}

	this.pos = pos;
	target.setValue(this.toTokenize, start, pos - start);
	return true;
}
 
Example 6
Source Project: flink   Source File: TranslateTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTranslateGraphIds() throws Exception {
	Graph<StringValue, LongValue, LongValue> stringIdGraph = graph
		.translateGraphIds(new LongValueToStringValue());

	for (Vertex<StringValue, LongValue> vertex : stringIdGraph.getVertices().collect()) {
		assertEquals(StringValue.class, vertex.f0.getClass());
		assertEquals(LongValue.class, vertex.f1.getClass());
	}

	for (Edge<StringValue, LongValue> edge : stringIdGraph.getEdges().collect()) {
		assertEquals(StringValue.class, edge.f0.getClass());
		assertEquals(StringValue.class, edge.f1.getClass());
		assertEquals(LongValue.class, edge.f2.getClass());
	}

	TestBaseUtils.compareResultAsText(stringIdGraph.getVertices().collect(), expectedVertexResult);
	TestBaseUtils.compareResultAsText(stringIdGraph.getEdges().collect(), expectedEdgeResult);
}
 
Example 7
Source Project: flink   Source File: DriverTestData.java    License: Apache License 2.0 6 votes vote down vote up
public static List<Tuple2<StringValue, IntValue>> createReduceMutableData() {
	List<Tuple2<StringValue, IntValue>> data = new ArrayList<Tuple2<StringValue, IntValue>>();
	
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("a"), new IntValue(1)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("b"), new IntValue(2)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("c"), new IntValue(3)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("d"), new IntValue(4)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("d"), new IntValue(5)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("e"), new IntValue(6)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("e"), new IntValue(7)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("e"), new IntValue(8)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(9)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(10)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(11)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(12)));
	
	return data;
}
 
Example 8
Source Project: flink   Source File: ValueArrayFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example 9
Source Project: flink   Source File: AccumulatorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void postSubmit() throws Exception {
	compareResultsByLinesInMemory(EXPECTED, resultPath);

	// Test accumulator results
	System.out.println("Accumulator results:");
	JobExecutionResult res = this.result;
	System.out.println(AccumulatorHelper.getResultsFormatted(res.getAllAccumulatorResults()));

	Assert.assertEquals(Integer.valueOf(3), res.getAccumulatorResult("num-lines"));
	Assert.assertEquals(Integer.valueOf(3), res.getIntCounterResult("num-lines"));

	Assert.assertEquals(Double.valueOf(getParallelism()), res.getAccumulatorResult("open-close-counter"));

	// Test histogram (words per line distribution)
	Map<Integer, Integer> dist = new HashMap<>();
	dist.put(1, 1); dist.put(2, 1); dist.put(3, 1);
	Assert.assertEquals(dist, res.getAccumulatorResult("words-per-line"));

	// Test distinct words (custom accumulator)
	Set<StringValue> distinctWords = new HashSet<>();
	distinctWords.add(new StringValue("one"));
	distinctWords.add(new StringValue("two"));
	distinctWords.add(new StringValue("three"));
	Assert.assertEquals(distinctWords, res.getAccumulatorResult("distinct-words"));
}
 
Example 10
Source Project: flink   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadInvalidContentsLenient() {
	try {
		final String fileContent = "abc|222|def|444\nkkz|777|888|hhg";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, IntValue.class, StringValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new StringValue(), new IntValue(), new StringValue(), new IntValue() };
		
		assertNotNull(format.nextRecord(values));
		assertNull(format.nextRecord(values));
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void readWithParseQuotedStrings() {
	try {
		final String fileContent = "\"ab\\\"c\"|\"def\"\n\"ghijk\"|\"abc\"";
		final FileInputSplit split = createTempFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, StringValue.class);
		format.enableQuotedStringParsing('"');

		format.configure(parameters);
		format.open(split);

		Value[] values = new Value[] { new StringValue(), new StringValue()};

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ab\\\"c", ((StringValue) values[0]).getValue());
		assertEquals("def", ((StringValue) values[1]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ghijk", ((StringValue) values[0]).getValue());
		assertEquals("abc", ((StringValue) values[1]).getValue());

	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example 12
Source Project: flink   Source File: ValueArrayTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public TypeSerializer<ValueArray<T>> createSerializer(ExecutionConfig executionConfig) {
	Preconditions.checkNotNull(type, "TypeInformation type class is required");

	if (ByteValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new ByteValueArraySerializer();
	} else if (CharValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new CharValueArraySerializer();
	} else if (DoubleValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new DoubleValueArraySerializer();
	} else if (FloatValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new FloatValueArraySerializer();
	} else if (IntValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new IntValueArraySerializer();
	} else if (LongValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new LongValueArraySerializer();
	} else if (NullValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new NullValueArraySerializer();
	} else if (ShortValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new ShortValueArraySerializer();
	} else if (StringValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new StringValueArraySerializer();
	} else {
		throw new InvalidTypesException("No ValueArray class exists for " + type);
	}
}
 
Example 13
Source Project: flink   Source File: AllGroupReduceDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
	Tuple2<StringValue, IntValue> current = new Tuple2<StringValue, IntValue>(new StringValue(""), new IntValue(0));
	
	for (Tuple2<StringValue, IntValue> next : values) {
		next.f0.append(current.f0);
		next.f1.setValue(current.f1.getValue() + next.f1.getValue());
		current = next;
	}
	
	out.collect(current);
}
 
Example 14
Source Project: Flink-CEPplus   Source File: ValueCollectionDataSets.java    License: Apache License 2.0 5 votes vote down vote up
public static DataSet<Tuple5<IntValue, LongValue, IntValue, StringValue, LongValue>> get5TupleDataSet(ExecutionEnvironment env) {
	List<Tuple5<IntValue, LongValue, IntValue, StringValue, LongValue>> data = new ArrayList<>();

	data.add(new Tuple5<>(new IntValue(1), new LongValue(1L), new IntValue(0), new StringValue("Hallo"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(2), new LongValue(2L), new IntValue(1), new StringValue("Hallo Welt"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(2), new LongValue(3L), new IntValue(2), new StringValue("Hallo Welt wie"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(3), new LongValue(4L), new IntValue(3), new StringValue("Hallo Welt wie gehts?"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(3), new LongValue(5L), new IntValue(4), new StringValue("ABC"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(3), new LongValue(6L), new IntValue(5), new StringValue("BCD"), new LongValue(3L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(7L), new IntValue(6), new StringValue("CDE"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(8L), new IntValue(7), new StringValue("DEF"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(9L), new IntValue(8), new StringValue("EFG"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(10L), new IntValue(9), new StringValue("FGH"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(11L), new IntValue(10), new StringValue("GHI"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(12L), new IntValue(11), new StringValue("HIJ"), new LongValue(3L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(13L), new IntValue(12), new StringValue("IJK"), new LongValue(3L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(14L), new IntValue(13), new StringValue("JKL"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(15L), new IntValue(14), new StringValue("KLM"), new LongValue(2L)));

	Collections.shuffle(data);

	TupleTypeInfo<Tuple5<IntValue, LongValue, IntValue, StringValue, LongValue>> type = new
		TupleTypeInfo<>(
			ValueTypeInfo.INT_VALUE_TYPE_INFO,
			ValueTypeInfo.LONG_VALUE_TYPE_INFO,
			ValueTypeInfo.INT_VALUE_TYPE_INFO,
			ValueTypeInfo.STRING_VALUE_TYPE_INFO,
			ValueTypeInfo.LONG_VALUE_TYPE_INFO
	);

	return env.fromCollection(data, type);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: ValueCollectionDataSets.java    License: Apache License 2.0 5 votes vote down vote up
public static DataSet<Tuple2<Tuple2<IntValue, IntValue>, StringValue>> getSmallNestedTupleDataSet(ExecutionEnvironment env) {
	List<Tuple2<Tuple2<IntValue, IntValue>, StringValue>> data = new ArrayList<>();

	data.add(new Tuple2<>(new Tuple2<>(new IntValue(1), new IntValue(1)), new StringValue("one")));
	data.add(new Tuple2<>(new Tuple2<>(new IntValue(2), new IntValue(2)), new StringValue("two")));
	data.add(new Tuple2<>(new Tuple2<>(new IntValue(3), new IntValue(3)), new StringValue("three")));

	TupleTypeInfo<Tuple2<Tuple2<IntValue, IntValue>, StringValue>> type = new
		TupleTypeInfo<>(
			new TupleTypeInfo<Tuple2<IntValue, IntValue>>(ValueTypeInfo.INT_VALUE_TYPE_INFO, ValueTypeInfo.INT_VALUE_TYPE_INFO),
			ValueTypeInfo.STRING_VALUE_TYPE_INFO
	);

	return env.fromCollection(data, type);
}
 
Example 16
Source Project: flink   Source File: CsvOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeRecord(T element) throws IOException {
	int numFields = element.getArity();

	for (int i = 0; i < numFields; i++) {
		Object v = element.getField(i);
		if (v != null) {
			if (i != 0) {
				this.wrt.write(this.fieldDelimiter);
			}

			if (quoteStrings) {
				if (v instanceof String || v instanceof StringValue) {
					this.wrt.write('"');
					this.wrt.write(v.toString());
					this.wrt.write('"');
				} else {
					this.wrt.write(v.toString());
				}
			} else {
				this.wrt.write(v.toString());
			}
		} else {
			if (this.allowNullValues) {
				if (i != 0) {
					this.wrt.write(this.fieldDelimiter);
				}
			} else {
				throw new RuntimeException("Cannot write tuple with <null> value at position: " + i);
			}
		}
	}

	// add the record delimiter
	this.wrt.write(this.recordDelimiter);
}
 
Example 17
Source Project: flink   Source File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 
Example 18
Source Project: flink   Source File: StringArraySerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int len = source.readInt();
	target.writeInt(len);
	
	for (int i = 0; i < len; i++) {
		StringValue.copyString(source, target);
	}
}
 
Example 19
Source Project: flink   Source File: AllGroupReduceDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAllReduceDriverMutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableReducer());
		
		AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Tuple2<StringValue, IntValue> res = result.getList().get(0);
		
		char[] foundString = res.f0.getValue().toCharArray();
		Arrays.sort(foundString);
		
		char[] expectedString = "abcddeeeffff".toCharArray();
		Arrays.sort(expectedString);
		
		Assert.assertArrayEquals(expectedString, foundString);
		Assert.assertEquals(78, res.f1.getValue());
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: CsvReaderITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testValueTypes() throws Exception {
	final String inputData = "ABC,true,1,2,3,4,5.0,6.0\nBCD,false,1,2,3,4,5.0,6.0";
	final String dataPath = createInputData(inputData);
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> data =
			env.readCsvFile(dataPath).types(StringValue.class, BooleanValue.class, ByteValue.class, ShortValue.class, IntValue.class, LongValue.class, FloatValue.class, DoubleValue.class);
	List<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> result = data.collect();

	expected = inputData;
	compareResultAsTuples(result, expected);
}
 
Example 21
Source Project: flink   Source File: StringValueComparatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected StringValue[] getSortedTestData() {
	return new StringValue[]{
		new StringValue(""),
		new StringValue("Lorem Ipsum Dolor Omit Longer"),
		new StringValue("aaaa"),
		new StringValue("abcd"),
		new StringValue("abce"),
		new StringValue("abdd"),
		new StringValue("accd"),
		new StringValue("bbcd")
	};
}
 
Example 22
Source Project: Flink-CEPplus   Source File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 
Example 23
Source Project: Flink-CEPplus   Source File: VarLengthStringParserTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testParseValidMixedStrings() {

	this.parser = new StringValueParser();
	this.parser.enableQuotedStringParsing((byte)'@');

	// check valid strings with out whitespaces and trailing delimiter
	byte[] recBytes = "@abcde|[email protected]|@[email protected]|jklmnopq|@[email protected]|tuv".getBytes(ConfigConstants.DEFAULT_CHARSET);
	StringValue s = new StringValue();

	int startPos = 0;
	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 11);
	assertTrue(s.getValue().equals("abcde|gh"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 15);
	assertTrue(s.getValue().equals("i"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 24);
	assertTrue(s.getValue().equals("jklmnopq"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 29);
	assertTrue(s.getValue().equals("rs"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 32);
	assertTrue(s.getValue().equals("tuv"));

}
 
Example 24
Source Project: Flink-CEPplus   Source File: DistinctTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void translateDistinctPosition() {
	try {
		final int parallelism = 8;
		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);

		DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);

		initialData.distinct(1, 2).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, LongValue>>());

		Plan p = env.createProgramPlan();

		GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();

		// currently distinct is translated to a Reduce
		ReduceOperatorBase<?, ?> reducer = (ReduceOperatorBase<?, ?>) sink.getInput();

		// check types
		assertEquals(initialData.getType(), reducer.getOperatorInfo().getInputType());
		assertEquals(initialData.getType(), reducer.getOperatorInfo().getOutputType());

		// check keys
		assertArrayEquals(new int[] {1, 2}, reducer.getKeyColumns(0));

		// parallelism was not configured on the operator
		assertTrue(reducer.getParallelism() == 1 || reducer.getParallelism() == -1);

		assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test caused an error: " + e.getMessage());
	}
}
 
Example 25
Source Project: Flink-CEPplus   Source File: StringValueUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Replaces all non-word characters in a string by a given character. The only
 * characters not replaced are the characters that qualify as word characters
 * or digit characters with respect to {@link Character#isLetter(char)} or
 * {@link Character#isDigit(char)}, as well as the underscore character.
 *
 * <p>This operation is intended to simplify strings for counting distinct words.
 *
 * @param string The string value to have the non-word characters replaced.
 * @param replacement The character to use as the replacement.
 */
public static void replaceNonWordChars(StringValue string, char replacement) {
	final char[] chars = string.getCharArray();
	final int len = string.length();

	for (int i = 0; i < len; i++) {
		final char c = chars[i];
		if (!(Character.isLetter(c) || Character.isDigit(c) || c == '_')) {
			chars[i] = replacement;
		}
	}
}
 
Example 26
Source Project: flink   Source File: DriverTestData.java    License: Apache License 2.0 5 votes vote down vote up
public static List<Tuple2<StringValue, IntValue>> createReduceMutableDataGroupedResult() {
	List<Tuple2<StringValue, IntValue>> data = new ArrayList<Tuple2<StringValue, IntValue>>();
	
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("a"), new IntValue(1)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("b"), new IntValue(2)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("c"), new IntValue(3)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("dd"), new IntValue(9)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("eee"), new IntValue(21)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("ffff"), new IntValue(42)));
	
	return data;
}
 
Example 27
Source Project: Flink-CEPplus   Source File: StringSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
public static final void testCopy(String[] values) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
	DataOutputStream serializer = new DataOutputStream(baos);
	
	for (String value : values) {
		StringValue.writeString(value, serializer);
	}
	
	serializer.close();
	baos.close();
	
	ByteArrayInputStream sourceInput = new ByteArrayInputStream(baos.toByteArray());
	DataInputStream source = new DataInputStream(sourceInput);
	ByteArrayOutputStream targetOutput = new ByteArrayOutputStream(4096);
	DataOutputStream target = new DataOutputStream(targetOutput);
	
	for (int i = 0; i < values.length; i++) {
		StringValue.copyString(source, target);
	}
	
	ByteArrayInputStream validateInput = new ByteArrayInputStream(targetOutput.toByteArray());
	DataInputStream validate = new DataInputStream(validateInput);
	
	int num = 0;
	while (validate.available() > 0) {
		String deser = StringValue.readString(validate);
		
		assertEquals("DeserializedString differs from original string.", values[num], deser);
		num++;
	}
	
	assertEquals("Wrong number of deserialized values", values.length, num);
}
 
Example 28
Source Project: Flink-CEPplus   Source File: CSVReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWithValueType() throws Exception {
	CsvReader reader = getCsvReader();
	DataSource<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> items =
			reader.types(StringValue.class, BooleanValue.class, ByteValue.class, ShortValue.class, IntValue.class, LongValue.class, FloatValue.class, DoubleValue.class);
	TypeInformation<?> info = items.getType();

	Assert.assertEquals(true, info.isTupleType());
	Assert.assertEquals(Tuple8.class, info.getTypeClass());
}
 
Example 29
Source Project: flink   Source File: GraphKeyTypeTransform.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LongValueWithProperHashCode translate(StringValue value, LongValueWithProperHashCode reuse)
		throws Exception {
	if (reuse == null) {
		reuse = new LongValueWithProperHashCode();
	}

	reuse.setValue(Long.parseLong(value.getValue()));
	return reuse;
}
 
Example 30
Source Project: flink   Source File: PairGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public PairGenerator(long seed, int keyMax, int valueLength, KeyMode keyMode, ValueMode valueMode, String constant) {
	this.seed = seed;
	this.keyMax = keyMax;
	this.valueLength = valueLength;
	this.keyMode = keyMode;
	this.valueMode = valueMode;

	this.random = new Random(seed);
	this.counter = 0;
	
	this.valueConstant = new StringValue();
	if (constant != null) {
		this.valueConstant.setValue(constant);
	}
}