Java Code Examples for org.apache.flink.api.common.typeinfo.Types

The following examples show how to use org.apache.flink.api.common.typeinfo.Types. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: LastValueWithRetractAggFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeInformation<GenericRowData> getAccumulatorType() {
	LogicalType[] fieldTypes = new LogicalType[] {
			fromTypeInfoToLogicalType(getResultType()),
			new BigIntType(),
			new TypeInformationRawType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)),
			new TypeInformationRawType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false))
	};

	String[] fieldNames = new String[] {
			"lastValue",
			"lastOrder",
			"valueToOrderMapView",
			"orderToValueMapView"
	};

	return (TypeInformation) new RowDataTypeInfo(fieldTypes, fieldNames);
}
 
Example 2
Source Project: flink   Source File: FirstValueWithRetractAggFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeInformation<GenericRow> getAccumulatorType() {
	LogicalType[] fieldTypes = new LogicalType[] {
			fromTypeInfoToLogicalType(getResultType()),
			new BigIntType(),
			new TypeInformationAnyType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)),
			new TypeInformationAnyType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false))
	};

	String[] fieldNames = new String[] {
			"firstValue",
			"firstOrder",
			"valueToOrderMapView",
			"orderToValueMapView"
	};

	return (TypeInformation) new BaseRowTypeInfo(fieldTypes, fieldNames);
}
 
Example 3
Source Project: Alink   Source File: DataSetUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Count number of records in the dataset.
 *
 * @return a dataset of one record, recording the number of records of [[dataset]]
 */
public static <T> DataSet<Long> count(DataSet<T> dataSet) {
    return dataSet
        .mapPartition(new MapPartitionFunction<T, Long>() {
            @Override
            public void mapPartition(Iterable<T> values, Collector<Long> out) throws Exception {
                long cnt = 0L;
                for (T v : values) {
                    cnt++;
                }
                out.collect(cnt);
            }
        })
        .name("count_dataset")
        .returns(Types.LONG)
        .reduce(new ReduceFunction<Long>() {
            @Override
            public Long reduce(Long value1, Long value2) throws Exception {
                return value1 + value2;
            }
        });
}
 
Example 4
Source Project: flink   Source File: TableUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get the columns from featureCols who are included in the <code>categoricalCols</code>, and the columns whose
 * types are string or boolean.
 *
 * <p>If <code>categoricalCols</code> is null, return all the categorical columns.
 *
 * <p>for example: In FeatureHasher which projects a number of categorical or numerical features
 * into a feature vector of a specified dimension needs to identify the categorical features. And
 * the column which is the string or boolean must be categorical. We need to find these columns as
 * categorical when user do not specify the types(categorical or numerical).
 *
 * @param tableSchema     TableSchema.
 * @param featureCols     the columns to chosen from.
 * @param categoricalCols the columns which are included in the final result whatever the types of them are. And it
 *                        must be a subset of featureCols.
 * @return the categoricalCols.
 */
public static String[] getCategoricalCols(
	TableSchema tableSchema, String[] featureCols, String[] categoricalCols) {
	if (null == featureCols) {
		return categoricalCols;
	}
	List<String> categoricalList = null == categoricalCols ? null : Arrays.asList(categoricalCols);
	List<String> featureList = Arrays.asList(featureCols);
	if (null != categoricalCols && !featureList.containsAll(categoricalList)) {
		throw new IllegalArgumentException("CategoricalCols must be included in featureCols!");
	}

	TypeInformation<?>[] featureColTypes = findColTypes(tableSchema, featureCols);
	List<String> res = new ArrayList<>();
	for (int i = 0; i < featureCols.length; i++) {
		boolean included = null != categoricalList && categoricalList.contains(featureCols[i]);
		if (included || Types.BOOLEAN == featureColTypes[i] || Types.STRING == featureColTypes[i]) {
			res.add(featureCols[i]);
		}
	}

	return res.toArray(new String[0]);
}
 
Example 5
Source Project: flink   Source File: KeyedStateInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = IOException.class)
public void testInvalidProcessReaderFunctionFails() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new InvalidReaderFunction();

	readInputSplit(split, userFunction);

	Assert.fail("KeyedStateReaderFunction did not fail on invalid RuntimeContext use");
}
 
Example 6
Source Project: Alink   Source File: VectorImputerMapperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMax() throws Exception {
    Row[] rows = new Row[]{
        Row.of(0L, "{\"selectedCol\":\"\\\"vec\\\"\",\"strategy\":\"\\\"max\\\"\"}", null),
        Row.of(1048576L, "[1.3333333333333333,2.0]", null)
    };

    List<Row> model = Arrays.asList(rows);

    TableSchema dataSchema = new TableSchema(
        new String[]{"vec"},
        new TypeInformation<?>[]{Types.STRING}
    );
    Params params = new Params();

    VectorImputerModelMapper mapper = new VectorImputerModelMapper(modelSchema, dataSchema, params);
    mapper.loadModel(model);

    assertEquals(mapper.map(Row.of(new DenseVector(new double[]{1.0, Double.NaN}))).getField(0),
            new DenseVector(new double[]{1.0, 2.0}));
}
 
Example 7
Source Project: Alink   Source File: StopWordsRemoverMapperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCaseSensitive() throws Exception {
    TableSchema schema = new TableSchema(new String[] {"sentence"}, new TypeInformation<?>[] {Types.STRING});

    Params params = new Params()
        .set(StopWordsRemoverParams.SELECTED_COL, "sentence")
        .set(StopWordsRemoverParams.CASE_SENSITIVE, true)
        .set(StopWordsRemoverParams.STOP_WORDS, new String[]{"Test"});

    StopWordsRemoverMapper mapper = new StopWordsRemoverMapper(schema, params);
    mapper.open();

    assertEquals(mapper.map(Row.of("This is a unit test for filtering stopWords")).getField(0),
        "This unit test filtering stopWords");
    assertEquals(mapper.map(Row.of("Filter stopWords test")).getField(0),
        "Filter stopWords test");
    assertEquals(mapper.getOutputSchema(), schema);
}
 
Example 8
Source Project: flink   Source File: FirstValueWithRetractAggFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericRowData createAccumulator() {
	// The accumulator schema:
	// firstValue: T
	// fistOrder: Long
	// valueToOrderMap: RawValueData<MapView<T, List<Long>>>
	// orderToValueMap: RawValueData<MapView<Long, List<T>>>
	GenericRowData acc = new GenericRowData(4);
	acc.setField(0, null);
	acc.setField(1, null);
	acc.setField(2, RawValueData.fromObject(
		new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG))));
	acc.setField(3, RawValueData.fromObject(
		new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType()))));
	return acc;
}
 
Example 9
Source Project: Flink-CEPplus   Source File: TableFormatFactoryBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaDerivationWithRowtime() {
	final Map<String, String> properties = new HashMap<>();
	properties.put("schema.0.name", "otherField");
	properties.put("schema.0.type", "VARCHAR");
	properties.put("schema.0.from", "csvField");
	properties.put("schema.1.name", "abcField");
	properties.put("schema.1.type", "VARCHAR");
	properties.put("schema.2.name", "p");
	properties.put("schema.2.type", "TIMESTAMP");
	properties.put("schema.2.proctime", "true");
	properties.put("schema.3.name", "r");
	properties.put("schema.3.type", "TIMESTAMP");
	properties.put("schema.3.rowtime.timestamps.type", "from-field"); // from-field strategy
	properties.put("schema.3.rowtime.timestamps.from", "myTime");
	properties.put("schema.3.rowtime.watermarks.type", "from-source");

	final TableSchema actualSchema = TableFormatFactoryBase.deriveSchema(properties);
	final TableSchema expectedSchema = TableSchema.builder()
		.field("csvField", Types.STRING) // aliased
		.field("abcField", Types.STRING)
		.field("myTime", Types.SQL_TIMESTAMP)
		.build();
	assertEquals(expectedSchema, actualSchema);
}
 
Example 10
Source Project: Alink   Source File: VectorAssemblerTest.java    License: Apache License 2.0 6 votes vote down vote up
AlgoOperator getData(boolean isBatch) {
	TableSchema schema = new TableSchema(
		new String[] {"id", "c0", "c1", "c2"},
		new TypeInformation <?>[] {Types.STRING, Types.STRING, Types.STRING, Types.STRING}
	);

	List <Row> rows = new ArrayList <>();

	rows.add(Row.of(new Object[] {"0", "$6$1:2.0 2:3.0 5:4.3", "3.0 2.0 3.0", "1 4 6 8"}));
	rows.add(Row.of(new Object[] {"1", "$8$1:2.0 2:3.0 7:4.3", "3.0 2.0 3.0", "1 4 6 8"}));
	rows.add(Row.of(new Object[] {"2", "$8$1:2.0 2:3.0 7:4.3", "2.0 3.0", "1 4 6 8"}));
	if (isBatch) {
		return new MemSourceBatchOp(rows, schema);
	} else {
		return new MemSourceStreamOp(rows, schema);
	}
}
 
Example 11
Source Project: Alink   Source File: ColumnsWriter.java    License: Apache License 2.0 6 votes vote down vote up
public ColumnsWriter(TableSchema schema) {
	this.nCols = schema.getFieldNames().length;
	this.colNames = schema.getFieldNames();
	this.isString = new boolean[colNames.length];
	TypeInformation[] fieldTypes = schema.getFieldTypes();

	this.parsers = new FieldParser[fieldTypes.length];

	for (int i = 0; i < fieldTypes.length; i++) {
		parsers[i] = getFieldParser(fieldTypes[i].getTypeClass());
		isString[i] = fieldTypes[i].equals(Types.STRING);
	}

	keyToFieldIdx = new HashMap <>();
	for (int i = 0; i < colNames.length; i++) {
		keyToFieldIdx.put(colNames[i], i);
	}
}
 
Example 12
Source Project: flink   Source File: TableFormatFactoryBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = new HashMap<>();
	properties.put("schema.0.name", "otherField");
	properties.put("schema.0.type", "VARCHAR");
	properties.put("schema.0.from", "csvField");
	properties.put("schema.1.name", "abcField");
	properties.put("schema.1.type", "VARCHAR");
	properties.put("schema.2.name", "p");
	properties.put("schema.2.type", "TIMESTAMP");
	properties.put("schema.2.proctime", "true");
	properties.put("schema.3.name", "r");
	properties.put("schema.3.type", "TIMESTAMP");
	properties.put("schema.3.rowtime.timestamps.type", "from-source");
	properties.put("schema.3.rowtime.watermarks.type", "from-source");

	final TableSchema actualSchema = TableFormatFactoryBase.deriveSchema(properties);
	final TableSchema expectedSchema = TableSchema.builder()
		.field("csvField", Types.STRING) // aliased
		.field("abcField", Types.STRING)
		.build();
	assertEquals(expectedSchema, actualSchema);
}
 
Example 13
Source Project: Alink   Source File: KMeansModelMapperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDetailDistanceOutput(){
    TableSchema dataSchema = new TableSchema(
        new String[] {"Y"}, new TypeInformation<?>[] {Types.STRING}
    );
    Params params = new Params()
        .set(KMeansPredictParams.PREDICTION_COL, "pred")
        .set(KMeansPredictParams.PREDICTION_DETAIL_COL, "detail")
        .set(KMeansPredictParams.PREDICTION_DISTANCE_COL, "distance");

    KMeansModelMapper mapper = new KMeansModelMapper(modelSchema, dataSchema, params);
    mapper.loadModel(model);

    Row res = mapper.map(Row.of("0 0 0"));
    assertEquals(res.getField(2), "0.010869565217391353 0.9891304347826086");
    assertEquals((double)res.getField(3), 0.173, 0.001);
    assertEquals(mapper.getOutputSchema(), new TableSchema(new String[] {"Y", "pred", "detail", "distance"},
        new TypeInformation<?>[] {Types.STRING, Types.LONG, Types.STRING, Types.DOUBLE}));
}
 
Example 14
Source Project: flink   Source File: TypeMappingUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFieldMappingLegacyCompositeTypeWithRenaming() {
	int[] indices = TypeMappingUtils.computePhysicalIndices(
		TableSchema.builder()
			.field("a", DataTypes.BIGINT())
			.field("b", DataTypes.STRING())
			.build().getTableColumns(),
		TypeConversions.fromLegacyInfoToDataType(Types.TUPLE(Types.STRING, Types.LONG)),
		str -> {
			switch (str) {
				case "a":
					return "f1";
				case "b":
					return "f0";
				default:
					throw new AssertionError();
			}
		}
	);

	assertThat(indices, equalTo(new int[]{1, 0}));
}
 
Example 15
Source Project: flink   Source File: TypeExtractorTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testFunctionWithMissingGenerics() {
	RichMapFunction function = new RichMapFunction() {
		private static final long serialVersionUID = 1L;

		@Override
		public String map(Object value) throws Exception {
			return null;
		}
	};

	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, Types.STRING, "name", true);
	Assert.assertTrue(ti instanceof MissingTypeInfo);
	
	try {
		TypeExtractor.getMapReturnTypes(function, Types.STRING);
		Assert.fail("Expected an exception");
	}
	catch (InvalidTypesException e) {
		// expected
	}
}
 
Example 16
Source Project: flink   Source File: JsonRowSchemaConverterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 
Example 17
Source Project: Alink   Source File: VectorEleWiseProdTest.java    License: Apache License 2.0 6 votes vote down vote up
AlgoOperator getData(boolean isBatch) {
	TableSchema schema = new TableSchema(
		new String[] {"id", "c0", "c1", "c2"},
		new TypeInformation <?>[] {Types.STRING, Types.STRING, Types.STRING, Types.STRING}
	);

	List <Row> rows = new ArrayList <>();
	rows.add(Row.of(new Object[] {"0", "$6$1:2.0 2:3.0 5:4.3", "3.0 2.0 3.0", "1 4 6 8"}));
	rows.add(Row.of(new Object[] {"1", "$8$1:2.0 2:3.0 7:4.3", "3.0 2.0 3.0", "1 4 6 8"}));
	rows.add(Row.of(new Object[] {"2", "$8$1:2.0 2:3.0 7:4.3", "2.0 3.0", "1 4 6 8"}));

	if (isBatch) {
		return new MemSourceBatchOp(rows, schema);
	} else {
		return new MemSourceStreamOp(rows, schema);
	}
}
 
Example 18
Source Project: Alink   Source File: EvalRegressionBatchOp.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public EvalRegressionBatchOp linkFrom(BatchOperator<?>... inputs) {
    BatchOperator in = checkAndGetFirst(inputs);

    TableUtil.findColIndexWithAssertAndHint(in.getColNames(), this.getLabelCol());
    TableUtil.findColIndexWithAssertAndHint(in.getColNames(), this.getPredictionCol());

    TableUtil.assertNumericalCols(in.getSchema(), this.getLabelCol(), this.getPredictionCol());
    DataSet<Row> out = in.select(new String[] {this.getLabelCol(), this.getPredictionCol()})
        .getDataSet()
        .rebalance()
        .mapPartition(new CalcLocal())
        .reduce(new EvaluationUtil.ReduceBaseMetrics())
        .flatMap(new EvaluationUtil.SaveDataAsParams());

    this.setOutputTable(DataSetConversionUtil.toTable(getMLEnvironmentId(),
        out, new TableSchema(new String[] {"regression_eval_result"}, new TypeInformation[] {Types.STRING})
    ));
    return this;
}
 
Example 19
Source Project: Alink   Source File: BisectingKMeansModelMapperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDetailOutput() throws Exception{
    TableSchema dataSchema = new TableSchema(
        new String[] {"Y"}, new TypeInformation<?>[] {Types.STRING}
    );
    Params params = new Params()
        .set(KMeansPredictParams.PREDICTION_COL, "pred")
        .set(KMeansPredictParams.PREDICTION_DETAIL_COL, "detail");

    BisectingKMeansModelMapper mapper = new BisectingKMeansModelMapper(modelSchema, dataSchema, params);
    mapper.loadModel(model);

    assertEquals(mapper.map(Row.of("0 0 0")).getField(2), "0.5 0.25 0.25");
    assertEquals(mapper.getOutputSchema(), new TableSchema(new String[] {"Y", "pred", "detail"},
        new TypeInformation<?>[] {Types.STRING, Types.LONG, Types.STRING}));
}
 
Example 20
Source Project: Alink   Source File: CsvFormatterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDoublePrecision() throws Exception {
    TypeInformation[] types = new TypeInformation[]{Types.DOUBLE};

    CsvFormatter formatter = new CsvFormatter(types, ",", '"');
    CsvParser parser = new CsvParser(types, ",", '"');

    Double[] values = new Double[]{Double.MAX_VALUE, Double.MIN_VALUE, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY,
        new Random().nextDouble()};
    for (Double v : values) {
        String text = formatter.format(Row.of(v));
        Row parsed = parser.parse(text).f1;
        Double p = (Double) parsed.getField(0);
        Assert.assertEquals(v, p, 0.);
    }
}
 
Example 21
Source Project: Alink   Source File: VectorAssemblerMapperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testToDense() throws Exception {
    TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"},
        new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING});

    TableSchema outSchema = new TableSchema(new String[]{"c0", "c1", "c2", "out"},
        new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING, VectorTypes.VECTOR});

    Params params = new Params()
        .set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"})
        .set(VectorAssemblerParams.OUTPUT_COL, "out");

    VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params);
    /* join the DenseVector, the number and the SparseVector together. the forth field shows the result */
    assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), 3.0, new SparseVector(3, new int[]{0, 2}, new double[]{1.0, 4.0}))).getField(3),
        new DenseVector(new double[]{3.0, 4.0, 3.0, 1.0, 0.0, 4.0}));
    assertEquals(mapper.getOutputSchema(), outSchema);
}
 
Example 22
Source Project: flink   Source File: FirstValueWithRetractAggFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericRow createAccumulator() {
	// The accumulator schema:
	// firstValue: T
	// fistOrder: Long
	// valueToOrderMap: BinaryGeneric<MapView<T, List<Long>>>
	// orderToValueMap: BinaryGeneric<MapView<Long, List<T>>>
	GenericRow acc = new GenericRow(4);
	acc.setField(0, null);
	acc.setField(1, null);
	acc.setField(2, new BinaryGeneric<>(
		new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)),
		getValueToOrderMapViewSerializer()));
	acc.setField(3, new BinaryGeneric<>(
		new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())),
		getOrderToValueMapViewSerializer()));
	return acc;
}
 
Example 23
Source Project: flink   Source File: JsonRowSchemaConverterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 
Example 24
Source Project: flink-learning   Source File: SQLExampleWordCount.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment blinkStreamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    blinkStreamEnv.setParallelism(1);
    EnvironmentSettings blinkStreamSettings = EnvironmentSettings.newInstance()
            .useBlinkPlanner()
            .inStreamingMode()
            .build();
    StreamTableEnvironment blinkStreamTableEnv = StreamTableEnvironment.create(blinkStreamEnv, blinkStreamSettings);

    String path = SQLExampleWordCount.class.getClassLoader().getResource("words.txt").getPath();

    CsvTableSource csvTableSource = CsvTableSource.builder()
            .field("word", Types.STRING)
            .path(path)
            .build();
    blinkStreamTableEnv.registerTableSource("zhisheng", csvTableSource);
    Table wordWithCount = blinkStreamTableEnv.sqlQuery("SELECT count(word), word FROM zhisheng GROUP BY word");
    blinkStreamTableEnv.toRetractStream(wordWithCount, Row.class).print();

    blinkStreamTableEnv.execute("Blink Stream SQL Job");
}
 
Example 25
@Test
public void testNestedSchema() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.ROW(Types.INT, Types.DOUBLE));

	final Row row = new Row(3);
	row.setField(0, 42);
	row.setField(1, false);
	final Row nested = new Row(2);
	nested.setField(0, 22);
	nested.setField(1, 2.3);
	row.setField(2, nested);

	final Row resultRow = serializeAndDeserialize(rowSchema, row);
	assertEquals(row, resultRow);
}
 
Example 26
Source Project: flink   Source File: TableUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void formatTest() {
	TableSchema tableSchema = new TableSchema(new String[] {"f0", "f1", "F2", "f3"},
		new TypeInformation[] {Types.INT, Types.LONG, Types.STRING, Types.BOOLEAN});
	Row row = Row.of(1, 2L, "3", true);

	String format = TableUtil.format(tableSchema.getFieldNames(), Collections.singletonList(row));
	Assert.assertTrue(("f0|f1|F2|f3\r\n" + "--|--|--|--\n" + "1|2|3|true").equalsIgnoreCase(format));
}
 
Example 27
Source Project: flink   Source File: TestRowDataCsvInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
private Object convertStringToInternal(String value, TypeInformation type) {
	if (type.equals(Types.INT)) {
		return Integer.parseInt(value);
	} else if (type.equals(Types.LONG)) {
		return Long.parseLong(value);
	} else if (type.equals(Types.STRING)) {
		return StringData.fromString(value);
	} else {
		throw new UnsupportedOperationException("Unsupported partition type: " + type);
	}
}
 
Example 28
Source Project: flink   Source File: OuterJoinRecordStateViews.java    License: Apache License 2.0 5 votes vote down vote up
private JoinKeyContainsUniqueKey(RuntimeContext ctx, String stateName, BaseRowTypeInfo recordType, StateTtlConfig ttlConfig) {
	TupleTypeInfo<Tuple2<BaseRow, Integer>> valueTypeInfo = new TupleTypeInfo<>(recordType, Types.INT);
	ValueStateDescriptor<Tuple2<BaseRow, Integer>> recordStateDesc = new ValueStateDescriptor<>(
		stateName,
		valueTypeInfo);
	if (!ttlConfig.equals(StateTtlConfig.DISABLED)) {
		recordStateDesc.enableTimeToLive(ttlConfig);
	}
	this.recordState = ctx.getState(recordStateDesc);
	// the result records always not more than 1
	this.reusedRecordList = new ArrayList<>(1);
	this.reusedTupleList = new ArrayList<>(1);
}
 
Example 29
Source Project: flink   Source File: BinaryHashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
	TypeInformation[] types = new TypeInformation[]{Types.INT, Types.INT};
	this.buildSideSerializer = new BinaryRowSerializer(types.length);
	this.probeSideSerializer = new BinaryRowSerializer(types.length);

	this.ioManager = new IOManagerAsync();

	conf = new Configuration();
	conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}
 
Example 30
Source Project: bahir-flink   Source File: StreamSchemaTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStreamSchemaWithTuple() {
    TypeInformation<Tuple4<Integer,Long,String,Double>> typeInfo = Types.TUPLE(Types.INT, Types.LONG, Types.STRING, Types.DOUBLE);
    StreamSchema<Tuple4<Integer,Long,String,Double>> schema = new StreamSchema<>(typeInfo, "id", "timestamp", "name", "price");
    assertEquals(Tuple4.class, schema.getTypeInfo().getTypeClass());
    assertEquals(4, schema.getFieldIndexes().length);
    assertEquals(Tuple4.class, schema.getTypeInfo().getTypeClass());
}