org.apache.flink.types.StringValue Java Examples

The following examples show how to use org.apache.flink.types.StringValue. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: flink-tpc-ds   File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNull(gFields);
	assertNull(order);
	assertNull(filtered.getUniqueFields());
}
 
Example #2
Source Project: flink   Author: flink-tpc-ds   File: StringValueArray.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String toString() {
	StringBuilder sb = new StringBuilder("[");
	String separator = "";

	for (StringValue sv : this) {
		sb
			.append(sv.getValue())
			.append(separator);
		separator = ",";
	}

	sb.append("]");

	return sb.toString();
}
 
Example #3
Source Project: flink   Author: flink-tpc-ds   File: AggregateITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFullAggregateOfMutableValueTypes() throws Exception {
	/*
	 * Full Aggregate of mutable value types
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<IntValue, LongValue, StringValue>> ds = ValueCollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple2<IntValue, LongValue>> aggregateDs = ds
			.aggregate(Aggregations.SUM, 0)
			.and(Aggregations.MAX, 1)
			.project(0, 1);

	List<Tuple2<IntValue, LongValue>> result = aggregateDs.collect();

	String expected = "231,6\n";

	compareResultAsTuples(result, expected);
}
 
Example #4
Source Project: flink   Author: apache   File: StringValueArray.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean addAll(ValueArray<StringValue> other) {
	StringValueArray source = (StringValueArray) other;

	int sourceSize = source.position;
	int newPosition = position + sourceSize;

	if (newPosition > data.length) {
		if (isBounded) {
			return false;
		} else {
			ensureCapacity(newPosition);
		}
	}

	System.arraycopy(source.data, 0, data, position, sourceSize);
	length += source.length;
 	    position = newPosition;

	return true;
}
 
Example #5
Source Project: flink   Author: apache   File: StringValueUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Gets the next token from the string. If another token is available, the token is stored
 * in the given target StringValue object.
 *
 * @param target The StringValue object to store the next token in.
 * @return True, if there was another token, false if not.
 */
public boolean next(StringValue target) {
	final char[] data = this.toTokenize.getCharArray();
	final int limit = this.limit;
	int pos = this.pos;

	// skip the delimiter
	for (; pos < limit && Character.isWhitespace(data[pos]); pos++) {
	}

	if (pos >= limit) {
		this.pos = pos;
		return false;
	}

	final int start = pos;
	for (; pos < limit && !Character.isWhitespace(data[pos]); pos++) {
	}

	this.pos = pos;
	target.setValue(this.toTokenize, start, pos - start);
	return true;
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: TranslateTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTranslateGraphIds() throws Exception {
	Graph<StringValue, LongValue, LongValue> stringIdGraph = graph
		.translateGraphIds(new LongValueToStringValue());

	for (Vertex<StringValue, LongValue> vertex : stringIdGraph.getVertices().collect()) {
		assertEquals(StringValue.class, vertex.f0.getClass());
		assertEquals(LongValue.class, vertex.f1.getClass());
	}

	for (Edge<StringValue, LongValue> edge : stringIdGraph.getEdges().collect()) {
		assertEquals(StringValue.class, edge.f0.getClass());
		assertEquals(StringValue.class, edge.f1.getClass());
		assertEquals(LongValue.class, edge.f2.getClass());
	}

	TestBaseUtils.compareResultAsText(stringIdGraph.getVertices().collect(), expectedVertexResult);
	TestBaseUtils.compareResultAsText(stringIdGraph.getEdges().collect(), expectedEdgeResult);
}
 
Example #7
Source Project: flink   Author: apache   File: DriverTestData.java    License: Apache License 2.0 6 votes vote down vote up
public static List<Tuple2<StringValue, IntValue>> createReduceMutableData() {
	List<Tuple2<StringValue, IntValue>> data = new ArrayList<Tuple2<StringValue, IntValue>>();
	
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("a"), new IntValue(1)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("b"), new IntValue(2)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("c"), new IntValue(3)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("d"), new IntValue(4)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("d"), new IntValue(5)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("e"), new IntValue(6)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("e"), new IntValue(7)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("e"), new IntValue(8)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(9)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(10)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(11)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("f"), new IntValue(12)));
	
	return data;
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: ValueArrayFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: AccumulatorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void postSubmit() throws Exception {
	compareResultsByLinesInMemory(EXPECTED, resultPath);

	// Test accumulator results
	System.out.println("Accumulator results:");
	JobExecutionResult res = this.result;
	System.out.println(AccumulatorHelper.getResultsFormatted(res.getAllAccumulatorResults()));

	Assert.assertEquals(Integer.valueOf(3), res.getAccumulatorResult("num-lines"));
	Assert.assertEquals(Integer.valueOf(3), res.getIntCounterResult("num-lines"));

	Assert.assertEquals(Double.valueOf(getParallelism()), res.getAccumulatorResult("open-close-counter"));

	// Test histogram (words per line distribution)
	Map<Integer, Integer> dist = new HashMap<>();
	dist.put(1, 1); dist.put(2, 1); dist.put(3, 1);
	Assert.assertEquals(dist, res.getAccumulatorResult("words-per-line"));

	// Test distinct words (custom accumulator)
	Set<StringValue> distinctWords = new HashSet<>();
	distinctWords.add(new StringValue("one"));
	distinctWords.add(new StringValue("two"));
	distinctWords.add(new StringValue("three"));
	Assert.assertEquals(distinctWords, res.getAccumulatorResult("distinct-words"));
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: GenericCsvInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadInvalidContentsLenient() {
	try {
		final String fileContent = "abc|222|def|444\nkkz|777|888|hhg";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, IntValue.class, StringValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new StringValue(), new IntValue(), new StringValue(), new IntValue() };
		
		assertNotNull(format.nextRecord(values));
		assertNull(format.nextRecord(values));
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example #11
Source Project: Flink-CEPplus   Author: ljygz   File: GenericCsvInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void readWithParseQuotedStrings() {
	try {
		final String fileContent = "\"ab\\\"c\"|\"def\"\n\"ghijk\"|\"abc\"";
		final FileInputSplit split = createTempFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, StringValue.class);
		format.enableQuotedStringParsing('"');

		format.configure(parameters);
		format.open(split);

		Value[] values = new Value[] { new StringValue(), new StringValue()};

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ab\\\"c", ((StringValue) values[0]).getValue());
		assertEquals("def", ((StringValue) values[1]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("ghijk", ((StringValue) values[0]).getValue());
		assertEquals("abc", ((StringValue) values[1]).getValue());

	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: ValueArrayTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public TypeSerializer<ValueArray<T>> createSerializer(ExecutionConfig executionConfig) {
	Preconditions.checkNotNull(type, "TypeInformation type class is required");

	if (ByteValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new ByteValueArraySerializer();
	} else if (CharValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new CharValueArraySerializer();
	} else if (DoubleValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new DoubleValueArraySerializer();
	} else if (FloatValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new FloatValueArraySerializer();
	} else if (IntValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new IntValueArraySerializer();
	} else if (LongValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new LongValueArraySerializer();
	} else if (NullValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new NullValueArraySerializer();
	} else if (ShortValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new ShortValueArraySerializer();
	} else if (StringValue.class.isAssignableFrom(type)) {
		return (TypeSerializer<ValueArray<T>>) (TypeSerializer<?>) new StringValueArraySerializer();
	} else {
		throw new InvalidTypesException("No ValueArray class exists for " + type);
	}
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: AllGroupReduceDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
	Tuple2<StringValue, IntValue> current = new Tuple2<StringValue, IntValue>(new StringValue(""), new IntValue(0));
	
	for (Tuple2<StringValue, IntValue> next : values) {
		next.f0.append(current.f0);
		next.f1.setValue(current.f1.getValue() + next.f1.getValue());
		current = next;
	}
	
	out.collect(current);
}
 
Example #14
Source Project: Flink-CEPplus   Author: ljygz   File: ValueCollectionDataSets.java    License: Apache License 2.0 5 votes vote down vote up
public static DataSet<Tuple5<IntValue, LongValue, IntValue, StringValue, LongValue>> get5TupleDataSet(ExecutionEnvironment env) {
	List<Tuple5<IntValue, LongValue, IntValue, StringValue, LongValue>> data = new ArrayList<>();

	data.add(new Tuple5<>(new IntValue(1), new LongValue(1L), new IntValue(0), new StringValue("Hallo"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(2), new LongValue(2L), new IntValue(1), new StringValue("Hallo Welt"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(2), new LongValue(3L), new IntValue(2), new StringValue("Hallo Welt wie"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(3), new LongValue(4L), new IntValue(3), new StringValue("Hallo Welt wie gehts?"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(3), new LongValue(5L), new IntValue(4), new StringValue("ABC"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(3), new LongValue(6L), new IntValue(5), new StringValue("BCD"), new LongValue(3L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(7L), new IntValue(6), new StringValue("CDE"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(8L), new IntValue(7), new StringValue("DEF"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(9L), new IntValue(8), new StringValue("EFG"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(4), new LongValue(10L), new IntValue(9), new StringValue("FGH"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(11L), new IntValue(10), new StringValue("GHI"), new LongValue(1L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(12L), new IntValue(11), new StringValue("HIJ"), new LongValue(3L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(13L), new IntValue(12), new StringValue("IJK"), new LongValue(3L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(14L), new IntValue(13), new StringValue("JKL"), new LongValue(2L)));
	data.add(new Tuple5<>(new IntValue(5), new LongValue(15L), new IntValue(14), new StringValue("KLM"), new LongValue(2L)));

	Collections.shuffle(data);

	TupleTypeInfo<Tuple5<IntValue, LongValue, IntValue, StringValue, LongValue>> type = new
		TupleTypeInfo<>(
			ValueTypeInfo.INT_VALUE_TYPE_INFO,
			ValueTypeInfo.LONG_VALUE_TYPE_INFO,
			ValueTypeInfo.INT_VALUE_TYPE_INFO,
			ValueTypeInfo.STRING_VALUE_TYPE_INFO,
			ValueTypeInfo.LONG_VALUE_TYPE_INFO
	);

	return env.fromCollection(data, type);
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: ValueCollectionDataSets.java    License: Apache License 2.0 5 votes vote down vote up
public static DataSet<Tuple2<Tuple2<IntValue, IntValue>, StringValue>> getSmallNestedTupleDataSet(ExecutionEnvironment env) {
	List<Tuple2<Tuple2<IntValue, IntValue>, StringValue>> data = new ArrayList<>();

	data.add(new Tuple2<>(new Tuple2<>(new IntValue(1), new IntValue(1)), new StringValue("one")));
	data.add(new Tuple2<>(new Tuple2<>(new IntValue(2), new IntValue(2)), new StringValue("two")));
	data.add(new Tuple2<>(new Tuple2<>(new IntValue(3), new IntValue(3)), new StringValue("three")));

	TupleTypeInfo<Tuple2<Tuple2<IntValue, IntValue>, StringValue>> type = new
		TupleTypeInfo<>(
			new TupleTypeInfo<Tuple2<IntValue, IntValue>>(ValueTypeInfo.INT_VALUE_TYPE_INFO, ValueTypeInfo.INT_VALUE_TYPE_INFO),
			ValueTypeInfo.STRING_VALUE_TYPE_INFO
	);

	return env.fromCollection(data, type);
}
 
Example #16
Source Project: flink   Author: apache   File: CsvOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeRecord(T element) throws IOException {
	int numFields = element.getArity();

	for (int i = 0; i < numFields; i++) {
		Object v = element.getField(i);
		if (v != null) {
			if (i != 0) {
				this.wrt.write(this.fieldDelimiter);
			}

			if (quoteStrings) {
				if (v instanceof String || v instanceof StringValue) {
					this.wrt.write('"');
					this.wrt.write(v.toString());
					this.wrt.write('"');
				} else {
					this.wrt.write(v.toString());
				}
			} else {
				this.wrt.write(v.toString());
			}
		} else {
			if (this.allowNullValues) {
				if (i != 0) {
					this.wrt.write(this.fieldDelimiter);
				}
			} else {
				throw new RuntimeException("Cannot write tuple with <null> value at position: " + i);
			}
		}
	}

	// add the record delimiter
	this.wrt.write(this.recordDelimiter);
}
 
Example #17
Source Project: flink   Author: apache   File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 
Example #18
Source Project: flink   Author: apache   File: StringArraySerializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int len = source.readInt();
	target.writeInt(len);
	
	for (int i = 0; i < len; i++) {
		StringValue.copyString(source, target);
	}
}
 
Example #19
Source Project: flink   Author: flink-tpc-ds   File: AllGroupReduceDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAllReduceDriverMutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableReducer());
		
		AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Tuple2<StringValue, IntValue> res = result.getList().get(0);
		
		char[] foundString = res.f0.getValue().toCharArray();
		Arrays.sort(foundString);
		
		char[] expectedString = "abcddeeeffff".toCharArray();
		Arrays.sort(expectedString);
		
		Assert.assertArrayEquals(expectedString, foundString);
		Assert.assertEquals(78, res.f1.getValue());
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: CsvReaderITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testValueTypes() throws Exception {
	final String inputData = "ABC,true,1,2,3,4,5.0,6.0\nBCD,false,1,2,3,4,5.0,6.0";
	final String dataPath = createInputData(inputData);
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> data =
			env.readCsvFile(dataPath).types(StringValue.class, BooleanValue.class, ByteValue.class, ShortValue.class, IntValue.class, LongValue.class, FloatValue.class, DoubleValue.class);
	List<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> result = data.collect();

	expected = inputData;
	compareResultAsTuples(result, expected);
}
 
Example #21
Source Project: flink   Author: apache   File: StringValueComparatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected StringValue[] getSortedTestData() {
	return new StringValue[]{
		new StringValue(""),
		new StringValue("Lorem Ipsum Dolor Omit Longer"),
		new StringValue("aaaa"),
		new StringValue("abcd"),
		new StringValue("abce"),
		new StringValue("abdd"),
		new StringValue("accd"),
		new StringValue("bbcd")
	};
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: NFAStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: VarLengthStringParserTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testParseValidMixedStrings() {

	this.parser = new StringValueParser();
	this.parser.enableQuotedStringParsing((byte)'@');

	// check valid strings with out whitespaces and trailing delimiter
	byte[] recBytes = "@abcde|[email protected]|@[email protected]|jklmnopq|@[email protected]|tuv".getBytes(ConfigConstants.DEFAULT_CHARSET);
	StringValue s = new StringValue();

	int startPos = 0;
	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 11);
	assertTrue(s.getValue().equals("abcde|gh"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 15);
	assertTrue(s.getValue().equals("i"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 24);
	assertTrue(s.getValue().equals("jklmnopq"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 29);
	assertTrue(s.getValue().equals("rs"));

	startPos = parser.parseField(recBytes, startPos, recBytes.length, new byte[] {'|'}, s);
	assertTrue(startPos == 32);
	assertTrue(s.getValue().equals("tuv"));

}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: DistinctTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void translateDistinctPosition() {
	try {
		final int parallelism = 8;
		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);

		DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);

		initialData.distinct(1, 2).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, LongValue>>());

		Plan p = env.createProgramPlan();

		GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();

		// currently distinct is translated to a Reduce
		ReduceOperatorBase<?, ?> reducer = (ReduceOperatorBase<?, ?>) sink.getInput();

		// check types
		assertEquals(initialData.getType(), reducer.getOperatorInfo().getInputType());
		assertEquals(initialData.getType(), reducer.getOperatorInfo().getOutputType());

		// check keys
		assertArrayEquals(new int[] {1, 2}, reducer.getKeyColumns(0));

		// parallelism was not configured on the operator
		assertTrue(reducer.getParallelism() == 1 || reducer.getParallelism() == -1);

		assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test caused an error: " + e.getMessage());
	}
}
 
Example #25
Source Project: Flink-CEPplus   Author: ljygz   File: StringValueUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Replaces all non-word characters in a string by a given character. The only
 * characters not replaced are the characters that qualify as word characters
 * or digit characters with respect to {@link Character#isLetter(char)} or
 * {@link Character#isDigit(char)}, as well as the underscore character.
 *
 * <p>This operation is intended to simplify strings for counting distinct words.
 *
 * @param string The string value to have the non-word characters replaced.
 * @param replacement The character to use as the replacement.
 */
public static void replaceNonWordChars(StringValue string, char replacement) {
	final char[] chars = string.getCharArray();
	final int len = string.length();

	for (int i = 0; i < len; i++) {
		final char c = chars[i];
		if (!(Character.isLetter(c) || Character.isDigit(c) || c == '_')) {
			chars[i] = replacement;
		}
	}
}
 
Example #26
Source Project: flink   Author: apache   File: DriverTestData.java    License: Apache License 2.0 5 votes vote down vote up
public static List<Tuple2<StringValue, IntValue>> createReduceMutableDataGroupedResult() {
	List<Tuple2<StringValue, IntValue>> data = new ArrayList<Tuple2<StringValue, IntValue>>();
	
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("a"), new IntValue(1)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("b"), new IntValue(2)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("c"), new IntValue(3)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("dd"), new IntValue(9)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("eee"), new IntValue(21)));
	data.add(new Tuple2<StringValue, IntValue>(new StringValue("ffff"), new IntValue(42)));
	
	return data;
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: StringSerializationTest.java    License: Apache License 2.0 5 votes vote down vote up
public static final void testCopy(String[] values) throws IOException {
	ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
	DataOutputStream serializer = new DataOutputStream(baos);
	
	for (String value : values) {
		StringValue.writeString(value, serializer);
	}
	
	serializer.close();
	baos.close();
	
	ByteArrayInputStream sourceInput = new ByteArrayInputStream(baos.toByteArray());
	DataInputStream source = new DataInputStream(sourceInput);
	ByteArrayOutputStream targetOutput = new ByteArrayOutputStream(4096);
	DataOutputStream target = new DataOutputStream(targetOutput);
	
	for (int i = 0; i < values.length; i++) {
		StringValue.copyString(source, target);
	}
	
	ByteArrayInputStream validateInput = new ByteArrayInputStream(targetOutput.toByteArray());
	DataInputStream validate = new DataInputStream(validateInput);
	
	int num = 0;
	while (validate.available() > 0) {
		String deser = StringValue.readString(validate);
		
		assertEquals("DeserializedString differs from original string.", values[num], deser);
		num++;
	}
	
	assertEquals("Wrong number of deserialized values", values.length, num);
}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: CSVReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWithValueType() throws Exception {
	CsvReader reader = getCsvReader();
	DataSource<Tuple8<StringValue, BooleanValue, ByteValue, ShortValue, IntValue, LongValue, FloatValue, DoubleValue>> items =
			reader.types(StringValue.class, BooleanValue.class, ByteValue.class, ShortValue.class, IntValue.class, LongValue.class, FloatValue.class, DoubleValue.class);
	TypeInformation<?> info = items.getType();

	Assert.assertEquals(true, info.isTupleType());
	Assert.assertEquals(Tuple8.class, info.getTypeClass());
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: GraphKeyTypeTransform.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LongValueWithProperHashCode translate(StringValue value, LongValueWithProperHashCode reuse)
		throws Exception {
	if (reuse == null) {
		reuse = new LongValueWithProperHashCode();
	}

	reuse.setValue(Long.parseLong(value.getValue()));
	return reuse;
}
 
Example #30
Source Project: flink   Author: apache   File: PairGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public PairGenerator(long seed, int keyMax, int valueLength, KeyMode keyMode, ValueMode valueMode, String constant) {
	this.seed = seed;
	this.keyMax = keyMax;
	this.valueLength = valueLength;
	this.keyMode = keyMode;
	this.valueMode = valueMode;

	this.random = new Random(seed);
	this.counter = 0;
	
	this.valueConstant = new StringValue();
	if (constant != null) {
		this.valueConstant.setValue(constant);
	}
}