org.apache.flink.api.common.functions.FlatMapFunction Java Examples
The following examples show how to use
org.apache.flink.api.common.functions.FlatMapFunction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkStreamingTransformTranslators.java From flink-dataflow with Apache License 2.0 | 6 votes |
@Override public void translateNode(Read.Unbounded<T> transform, FlinkStreamingTranslationContext context) { PCollection<T> output = context.getOutput(transform); DataStream<WindowedValue<T>> source; if (transform.getSource().getClass().equals(UnboundedFlinkSource.class)) { UnboundedFlinkSource flinkSource = (UnboundedFlinkSource) transform.getSource(); source = context.getExecutionEnvironment() .addSource(flinkSource.getFlinkSource()) .flatMap(new FlatMapFunction<String, WindowedValue<String>>() { @Override public void flatMap(String s, Collector<WindowedValue<String>> collector) throws Exception { collector.collect(WindowedValue.<String>of(s, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)); } }).assignTimestampsAndWatermarks(new IngestionTimeExtractor()); } else { source = context.getExecutionEnvironment() .addSource(new UnboundedSourceWrapper<>(context.getPipelineOptions(), transform)); } context.setOutputDataStream(output, source); }
Example #2
Source File: FlatMapOperator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "FlatMap at " + defaultName; // create operator FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
Example #3
Source File: FilterOperator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected org.apache.flink.api.common.operators.base.FilterOperatorBase<T, FlatMapFunction<T, T>> translateToDataFlow(Operator<T> input) { String name = getName() != null ? getName() : "Filter at " + defaultName; // create operator PlanFilterOperator<T> po = new PlanFilterOperator<T>(function, name, getInputType()); po.setInput(input); // set parallelism if (getParallelism() > 0) { // use specified parallelism po.setParallelism(getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
Example #4
Source File: FilterOperator.java From flink with Apache License 2.0 | 6 votes |
@Override protected org.apache.flink.api.common.operators.base.FilterOperatorBase<T, FlatMapFunction<T, T>> translateToDataFlow(Operator<T> input) { String name = getName() != null ? getName() : "Filter at " + defaultName; // create operator PlanFilterOperator<T> po = new PlanFilterOperator<T>(function, name, getInputType()); po.setInput(input); // set parallelism if (getParallelism() > 0) { // use specified parallelism po.setParallelism(getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
Example #5
Source File: FlatMapOperator.java From flink with Apache License 2.0 | 6 votes |
@Override protected FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "FlatMap at " + defaultName; // create operator FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
Example #6
Source File: Main.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setGlobalJobParameters(params); DataSource<String> dataSource = env.fromElements(WORDS); dataSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() { @Override public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception { String[] words = line.split("\\W+"); for (String word : words) { out.collect(new Tuple2<>(word, 1)); } } }) .groupBy(0) .sum(1) .print(); long count = dataSource.count(); System.out.println(count); }
Example #7
Source File: GraphLoader.java From OSTMap with Apache License 2.0 | 6 votes |
private DataSet<Tuple2<String, UserNodeValues>> getUserNodes(DataSet<JSONObject> jsonData) { DataSet<Tuple2<String, UserNodeValues>> userNodes = jsonData.flatMap(new FlatMapFunction<JSONObject, Tuple2<String, UserNodeValues>>() { @Override public void flatMap(JSONObject jsonObject, Collector<Tuple2<String, UserNodeValues>> out) throws Exception { JSONObject user = jsonObject.getJSONObject("user"); String userId = user.getString("id_str"); String userName = user.getString("name"); out.collect(new Tuple2<String, UserNodeValues>(userId, new UserNodeValues(userId,userName))); // other mentioned users JSONObject entities = jsonObject.getJSONObject("entities"); JSONArray userMentions = entities.getJSONArray("user_mentions"); for (int i = 0; i < userMentions.length(); i++) { JSONObject current = userMentions.getJSONObject(i); String oUserId = current.getString("id_str"); String oUserName = current.getString("name"); out.collect(new Tuple2<String, UserNodeValues>(oUserId, new UserNodeValues(oUserId,oUserName))); } } }).distinct(0); return userNodes; }
Example #8
Source File: KafkaDeserializationSchemaTest.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args); StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool); Properties props = buildKafkaProps(parameterTool); FlinkKafkaConsumer011<ObjectNode> kafkaConsumer = new FlinkKafkaConsumer011<>("zhisheng", new KafkaMetricSchema(true), props); env.addSource(kafkaConsumer) .flatMap(new FlatMapFunction<ObjectNode, MetricEvent>() { @Override public void flatMap(ObjectNode jsonNodes, Collector<MetricEvent> collector) throws Exception { try { // System.out.println(jsonNodes); MetricEvent metricEvent = GsonUtil.fromJson(jsonNodes.get("value").asText(), MetricEvent.class); collector.collect(metricEvent); } catch (Exception e) { log.error("jsonNodes = {} convert to MetricEvent has an error", jsonNodes, e); } } }) .print(); env.execute(); }
Example #9
Source File: Main.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setGlobalJobParameters(params); DataSource<String> dataSource = env.fromElements(WORDS); dataSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() { @Override public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception { String[] words = line.split("\\W+"); for (String word : words) { out.collect(new Tuple2<>(word, 1)); } } }) .groupBy(0) .sum(1) .print(); long count = dataSource.count(); System.out.println(count); }
Example #10
Source File: KafkaDeserializationSchemaTest.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args); StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool); Properties props = buildKafkaProps(parameterTool); FlinkKafkaConsumer011<ObjectNode> kafkaConsumer = new FlinkKafkaConsumer011<>("zhisheng", new KafkaMetricSchema(true), props); env.addSource(kafkaConsumer) .flatMap(new FlatMapFunction<ObjectNode, MetricEvent>() { @Override public void flatMap(ObjectNode jsonNodes, Collector<MetricEvent> collector) throws Exception { try { // System.out.println(jsonNodes); MetricEvent metricEvent = GsonUtil.fromJson(jsonNodes.get("value").asText(), MetricEvent.class); collector.collect(metricEvent); } catch (Exception e) { log.error("jsonNodes = {} convert to MetricEvent has an error", jsonNodes, e); } } }) .print(); env.execute(); }
Example #11
Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 6 votes |
static FlatMapFunction<Event, String> createSemanticsCheckMapper(ParameterTool pt) { String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue()); SemanticsCheckMapper.ValidatorFunction validatorFunction; if (semantics.equalsIgnoreCase("exactly-once")) { validatorFunction = SemanticsCheckMapper.ValidatorFunction.exactlyOnce(); } else if (semantics.equalsIgnoreCase("at-least-once")) { validatorFunction = SemanticsCheckMapper.ValidatorFunction.atLeastOnce(); } else { throw new IllegalArgumentException("Unknown semantics requested: " + semantics); } return new SemanticsCheckMapper(validatorFunction); }
Example #12
Source File: AlsPredictBatchOp.java From Alink with Apache License 2.0 | 6 votes |
private static DataSet<Tuple2<Long, float[]>> getFactors(BatchOperator<?> model, final int identity) { return model.getDataSet() .flatMap(new FlatMapFunction<Row, Tuple2<Long, float[]>>() { @Override public void flatMap(Row value, Collector<Tuple2<Long, float[]>> out) throws Exception { int w = AlsModelDataConverter.getIsUser(value) ? 0 : 1; if (w != identity) { return; } long idx = AlsModelDataConverter.getVertexId(value); float[] factors = AlsModelDataConverter.getFactors(value); out.collect(Tuple2.of(idx, factors)); } }); }
Example #13
Source File: FlatMapOperator.java From flink with Apache License 2.0 | 6 votes |
@Override protected FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "FlatMap at " + defaultName; // create operator FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
Example #14
Source File: FlatMapOperatorBase.java From flink with Apache License 2.0 | 6 votes |
@Override protected List<OUT> executeOnCollections(List<IN> input, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { FlatMapFunction<IN, OUT> function = userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, ctx); FunctionUtils.openFunction(function, parameters); ArrayList<OUT> result = new ArrayList<OUT>(input.size()); TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig); TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig); CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer); for (IN element : input) { IN inCopy = inSerializer.copy(element); function.flatMap(inCopy, resultCollector); } FunctionUtils.closeFunction(function); return result; }
Example #15
Source File: AssociationRule.java From Alink with Apache License 2.0 | 6 votes |
/** * Generate association rules from frequent patterns. * * @param patterns A DataSet of frequent patterns and there supports. * @param transactionsCnt The number of transactions in the original dataset. * @param itemCounts A DataSet of items and their supports. * @param minConfidence Minimum confidence. * @param minLift Minimum lift. * @param maxConsequentLength Maximum length of a consequent. * @return The association rules with fields: antecedent(left hand side), consequent(right hand side), * support count, [lift, support, confidence]). */ public static DataSet<Tuple4<int[], int[], Integer, double[]>> extractRules( DataSet<Tuple2<int[], Integer>> patterns, DataSet<Long> transactionsCnt, DataSet<Tuple2<Integer, Integer>> itemCounts, final double minConfidence, final double minLift, final int maxConsequentLength) { if (maxConsequentLength <= 0) { return patterns.getExecutionEnvironment().fromElements(0) .flatMap(new FlatMapFunction<Integer, Tuple4<int[], int[], Integer, double[]>>() { @Override public void flatMap(Integer value, Collector<Tuple4<int[], int[], Integer, double[]>> out) throws Exception { } }); } else if (maxConsequentLength == 1) { return extractSingleConsequentRules(patterns, transactionsCnt, itemCounts, minConfidence, minLift); } else { return extractMultiConsequentRules(patterns, transactionsCnt, minConfidence, minLift, maxConsequentLength); } }
Example #16
Source File: DatahubSourceFunctionExample.java From alibaba-flink-connectors with Apache License 2.0 | 5 votes |
public void runExample() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DatahubSourceFunction datahubSource = new DatahubSourceFunction(endPoint, projectName, topicName, accessId, accessKey, 0, Long.MAX_VALUE, 1, 1, 1); env.addSource(datahubSource).flatMap( (FlatMapFunction<List<RecordEntry>, Tuple2<String, Long>>) (recordEntries, collector) -> { for (RecordEntry recordEntry : recordEntries) { collector.collect(getStringLongTuple2(recordEntry)); } }).returns(new TypeHint<Tuple2<String, Long>>() {}).print(); env.execute(); }
Example #17
Source File: KafkaConsumerTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Test that ensures that DeserializationSchema.isEndOfStream() is properly evaluated. * * @throws Exception */ public void runEndOfStreamTest() throws Exception { final int elementCount = 300; final String topic = writeSequence("testEndOfStream", elementCount, 1, 1); // read using custom schema final StreamExecutionEnvironment env1 = StreamExecutionEnvironment.getExecutionEnvironment(); env1.setParallelism(1); env1.getConfig().setRestartStrategy(RestartStrategies.noRestart()); env1.getConfig().disableSysoutLogging(); Properties props = new Properties(); props.putAll(standardProps); props.putAll(secureProps); DataStream<Tuple2<Integer, Integer>> fromKafka = env1.addSource(kafkaServer.getConsumer(topic, new FixedNumberDeserializationSchema(elementCount), props)); fromKafka.flatMap(new FlatMapFunction<Tuple2<Integer, Integer>, Void>() { @Override public void flatMap(Tuple2<Integer, Integer> value, Collector<Void> out) throws Exception { // noop ;) } }); tryExecute(env1, "Consume " + elementCount + " elements from Kafka"); deleteTestTopic(topic); }
Example #18
Source File: SampleITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private FlatMapOperator<Tuple3<Integer, Long, String>, String> getSourceDataSet(ExecutionEnvironment env) { return CollectionDataSets.get3TupleDataSet(env).flatMap( new FlatMapFunction<Tuple3<Integer, Long, String>, String>() { @Override public void flatMap(Tuple3<Integer, Long, String> value, Collector<String> out) throws Exception { out.collect(value.f2); } }); }
Example #19
Source File: ContinuousFileProcessingCheckpointITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void testProgram(StreamExecutionEnvironment env) { // set the restart strategy. env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(NO_OF_RETRIES, 0)); env.enableCheckpointing(10); // create and start the file creating thread. fc = new FileCreator(); fc.start(); // create the monitoring source along with the necessary readers. TextInputFormat format = new TextInputFormat(new org.apache.flink.core.fs.Path(localFsURI)); format.setFilesFilter(FilePathFilter.createDefaultFilter()); DataStream<String> inputStream = env.readFile(format, localFsURI, FileProcessingMode.PROCESS_CONTINUOUSLY, INTERVAL); TestingSinkFunction sink = new TestingSinkFunction(); inputStream.flatMap(new FlatMapFunction<String, String>() { @Override public void flatMap(String value, Collector<String> out) throws Exception { out.collect(value); } }).addSink(sink).setParallelism(1); }
Example #20
Source File: IPv6HostnamesITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testClusterWithIPv6host() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); env.getConfig().disableSysoutLogging(); // get input data DataSet<String> text = env.fromElements(WordCountData.TEXT.split("\n")); DataSet<Tuple2<String, Integer>> counts = text .flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() { @Override public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception { for (String token : value.toLowerCase().split("\\W+")) { if (token.length() > 0) { out.collect(new Tuple2<String, Integer>(token, 1)); } } } }) .groupBy(0).sum(1); List<Tuple2<String, Integer>> result = counts.collect(); TestBaseUtils.compareResultAsText(result, WordCountData.COUNTS_AS_TUPLES); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #21
Source File: StreamingOperatorsITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testOperatorChainWithObjectReuseAndNoOutputOperators() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableObjectReuse(); DataStream<Integer> input = env.fromElements(1, 2, 3); input.flatMap(new FlatMapFunction<Integer, Integer>() { @Override public void flatMap(Integer value, Collector<Integer> out) throws Exception { out.collect(value << 1); } }); env.execute(); }
Example #22
Source File: FlatMapOperatorCollectionTest.java From flink with Apache License 2.0 | 5 votes |
private FlatMapOperatorBase<String, String, FlatMapFunction<String, String>> getTestFlatMapOperator( FlatMapFunction<String, String> udf) { UnaryOperatorInformation<String, String> typeInfo = new UnaryOperatorInformation<String, String>( BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); return new FlatMapOperatorBase<String, String, FlatMapFunction<String, String>>( udf, typeInfo, "flatMap on Collections"); }
Example #23
Source File: ExactTriangleCount.java From gelly-streaming with Apache License 2.0 | 5 votes |
@SuppressWarnings("serial") private static SimpleEdgeStream<Integer, NullValue> getGraphStream(StreamExecutionEnvironment env) { if (fileOutput) { return new SimpleEdgeStream<>(env.readTextFile(edgeInputPath) .flatMap(new FlatMapFunction<String, Edge<Integer, NullValue>>() { @Override public void flatMap(String s, Collector<Edge<Integer, NullValue>> out) { String[] fields = s.split("\\s"); if (!fields[0].equals("%")) { int src = Integer.parseInt(fields[0]); int trg = Integer.parseInt(fields[1]); out.collect(new Edge<>(src, trg, NullValue.getInstance())); } } }), env); } return new SimpleEdgeStream<>(env.fromElements( new Edge<>(1, 2, NullValue.getInstance()), new Edge<>(2, 3, NullValue.getInstance()), new Edge<>(2, 6, NullValue.getInstance()), new Edge<>(5, 6, NullValue.getInstance()), new Edge<>(1, 4, NullValue.getInstance()), new Edge<>(5, 3, NullValue.getInstance()), new Edge<>(3, 4, NullValue.getInstance()), new Edge<>(3, 6, NullValue.getInstance()), new Edge<>(1, 3, NullValue.getInstance())), env); }
Example #24
Source File: WordCountWithAnonymousClass.java From flink with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { // set up the execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // get input data DataSet<String> text = StaticData.getDefaultTextLineDataSet(env); DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1) text.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() { @Override public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception { // normalize and split the line String[] tokens = value.toLowerCase().split("\\W+"); // emit the pairs for (String token : tokens) { if (token.length() > 0) { out.collect(new Tuple2<String, Integer>(token, 1)); } } } }) // group by the tuple field "0" and sum up tuple field "1" .groupBy(0) .sum(1); // emit result counts.print(); // execute program env.execute("WordCount Example"); }
Example #25
Source File: LookupJoinRunner.java From flink with Apache License 2.0 | 5 votes |
public LookupJoinRunner( GeneratedFunction<FlatMapFunction<RowData, RowData>> generatedFetcher, GeneratedCollector<TableFunctionCollector<RowData>> generatedCollector, boolean isLeftOuterJoin, int tableFieldsCount) { this.generatedFetcher = generatedFetcher; this.generatedCollector = generatedCollector; this.isLeftOuterJoin = isLeftOuterJoin; this.tableFieldsCount = tableFieldsCount; }
Example #26
Source File: ChainTaskTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testBatchTaskOutputInCloseMethod() { final int numChainedTasks = 10; final int keyCnt = 100; final int valCnt = 10; try { initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE); addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0); addOutput(outList); registerTask(FlatMapDriver.class, MockMapStub.class); for (int i = 0; i < numChainedTasks; i++) { final TaskConfig taskConfig = new TaskConfig(new Configuration()); taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD); taskConfig.setOutputSerializer(serFact); taskConfig.setStubWrapper( new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class)); getTaskConfig().addChainedTask( ChainedFlatMapDriver.class, taskConfig, "chained-" + i); } final BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv); testTask.invoke(); Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size()); } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #27
Source File: LambdaExtractionTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testFlatMapLambda() { FlatMapFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, String>> f = (i, out) -> out.collect(null); TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, null, true); if (!(ti instanceof MissingTypeInfo)) { assertTrue(ti.isTupleType()); assertEquals(2, ti.getArity()); assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType()); assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO); } }
Example #28
Source File: FlinkStreamingTransformTranslators.java From flink-dataflow with Apache License 2.0 | 5 votes |
@Override public void translateNode(TextIO.Write.Bound<T> transform, FlinkStreamingTranslationContext context) { PValue input = context.getInput(transform); DataStream<WindowedValue<T>> inputDataStream = context.getInputDataStream(input); String filenamePrefix = transform.getFilenamePrefix(); String filenameSuffix = transform.getFilenameSuffix(); boolean needsValidation = transform.needsValidation(); int numShards = transform.getNumShards(); String shardNameTemplate = transform.getShardNameTemplate(); // TODO: Implement these. We need Flink support for this. LOG.warn("Translation of TextIO.Write.needsValidation not yet supported. Is: {}.", needsValidation); LOG.warn("Translation of TextIO.Write.filenameSuffix not yet supported. Is: {}.", filenameSuffix); LOG.warn("Translation of TextIO.Write.shardNameTemplate not yet supported. Is: {}.", shardNameTemplate); DataStream<String> dataSink = inputDataStream.flatMap(new FlatMapFunction<WindowedValue<T>, String>() { @Override public void flatMap(WindowedValue<T> value, Collector<String> out) throws Exception { out.collect(value.getValue().toString()); } }); DataStreamSink<String> output = dataSink.writeAsText(filenamePrefix, FileSystem.WriteMode.OVERWRITE); if (numShards > 0) { output.setParallelism(numShards); } }
Example #29
Source File: FlatMapOperatorCollectionTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private FlatMapOperatorBase<String, String, FlatMapFunction<String, String>> getTestFlatMapOperator( FlatMapFunction<String, String> udf) { UnaryOperatorInformation<String, String> typeInfo = new UnaryOperatorInformation<String, String>( BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); return new FlatMapOperatorBase<String, String, FlatMapFunction<String, String>>( udf, typeInfo, "flatMap on Collections"); }
Example #30
Source File: ChainedFlatMapDriver.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void setup(AbstractInvokable parent) { @SuppressWarnings("unchecked") final FlatMapFunction<IT, OT> mapper = BatchTask.instantiateUserCode(this.config, userCodeClassLoader, FlatMapFunction.class); this.mapper = mapper; FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext()); }