org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableMap Java Examples

The following examples show how to use org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableMap. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReaderCheckpointHookTest.java    From flink-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void testRestore() throws Exception {
    ReaderGroup readerGroup = mock(ReaderGroup.class);

    Checkpoint checkpoint = mock(Checkpoint.class);
    CheckpointImpl checkpointImpl = mock(CheckpointImpl.class);

    when(checkpoint.asImpl()).thenReturn(checkpointImpl);
    when(checkpointImpl.getPositions()).thenReturn(ImmutableMap.<Stream, StreamCut>builder()
            .put(Stream.of(SCOPE, "s1"), getStreamCut("s1"))
            .put(Stream.of(SCOPE, "s2"), getStreamCut("s2")).build());

    ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
            .disableAutomaticCheckpoints()
            .startFromCheckpoint(checkpoint)
            .build();

    TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1),
            readerGroupConfig);

    hook.restoreCheckpoint(1L, checkpoint);

    verify(readerGroup).resetReaderGroup(readerGroupConfig);
}
 
Example #2
Source File: TestSource.java    From sylph with Apache License 2.0 6 votes vote down vote up
@Override
public void run(SourceContext<Row> sourceContext)
        throws Exception
{
    Random random = new Random();
    int numKeys = 10;
    long count = 1L;
    while (running) {
        long eventTime = System.currentTimeMillis() - random.nextInt(10 * 1000); //表示数据已经产生了 但是会有10秒以内的延迟
        String userId = "uid_" + count;

        String msg = MAPPER.writeValueAsString(ImmutableMap.of("user_id", userId, "ip", "127.0.0.1", "store", 12.0));
        Row row = Row.of("key" + random.nextInt(10), msg, eventTime);
        sourceContext.collect(row);
        count = count > numKeys ? 1L : count + 1;
        TimeUnit.MILLISECONDS.sleep(100);
    }
}
 
Example #3
Source File: JsonPathUdfTest.java    From sylph with Apache License 2.0 6 votes vote down vote up
@Before
public void init()
        throws JsonProcessingException
{
    String json = MAPPER.writeValueAsString(ImmutableMap.of("user_id", "uid_001",
            "ip", "127.0.0.1",
            "store", 12.0,
            "key1", ImmutableMap.of("key2", 123)
    ));

    StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment();
    execEnv.setParallelism(2);
    tableEnv = StreamTableEnvironment.create(execEnv);
    tableEnv.registerFunction("get_json_object", new UDFJson());
    table = tableEnv.sqlQuery("select '" + json + "' as message");
}
 
Example #4
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws IOException {
	File jarWithNoManifest = createJarFileWithManifest(ImmutableMap.of());
	File jarFile = TestJob.getTestJobJar();

	JarManifestParser.JarFileWithEntryClass jarFileWithEntryClass = JarManifestParser
		.findOnlyEntryClass(ImmutableList.of(jarWithNoManifest, jarFile));

	assertThat(jarFileWithEntryClass.getEntryClass(), is(equalTo(TestJob.class.getCanonicalName())));
}
 
Example #5
Source File: JarManifestParserTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassNoEntry() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of());

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertFalse(entry.isPresent());
}
 
Example #6
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassAssemblerClassAndMainClass() throws IOException {
	// We want the assembler class entry to have precedence over main class
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass",
		PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("AssemblerClass")));
}
 
Example #7
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassMainClass() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("MainClass")));
}
 
Example #8
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassAssemblerClass() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("AssemblerClass")));
}
 
Example #9
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassNoEntry() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of());

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertFalse(entry.isPresent());
}
 
Example #10
Source File: LeastUtilizationSlotMatchingStrategyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void findMatchingSlot_multipleMatchingSlots_returnsSlotWithLeastUtilization() {
	final ResourceProfile requestedResourceProfile = ResourceProfile.fromResources(2.0, 2);

	final TestingTaskManagerSlotInformation leastUtilizedSlot = TestingTaskManagerSlotInformation.newBuilder()
		.setResourceProfile(requestedResourceProfile)
		.build();
	final TestingTaskManagerSlotInformation tooSmallSlot = TestingTaskManagerSlotInformation.newBuilder()
		.setResourceProfile(ResourceProfile.fromResources(1.0, 10))
		.build();
	final TestingTaskManagerSlotInformation alternativeSlot = TestingTaskManagerSlotInformation.newBuilder()
		.setResourceProfile(requestedResourceProfile)
		.build();

	final Collection<TestingTaskManagerSlotInformation> freeSlots = Arrays.asList(
		tooSmallSlot,
		leastUtilizedSlot,
		alternativeSlot);

	Map<InstanceID, Integer> registeredSlotPerTaskExecutor = ImmutableMap.of(
		leastUtilizedSlot.getInstanceId(), 1,
		tooSmallSlot.getInstanceId(), 1,
		alternativeSlot.getInstanceId(), 2);

	final Optional<TestingTaskManagerSlotInformation> matchingSlot = LeastUtilizationSlotMatchingStrategy.INSTANCE.findMatchingSlot(
		requestedResourceProfile,
		freeSlots,
		createRegisteredSlotsLookupFunction(registeredSlotPerTaskExecutor));

	assertTrue(matchingSlot.isPresent());
	assertThat(matchingSlot.get().getSlotId(), is(leastUtilizedSlot.getSlotId()));
}
 
Example #11
Source File: SlotPoolImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCalculationOfTaskExecutorUtilization() throws Exception {
	try (final SlotPoolImpl slotPool = createSlotPoolImpl()) {
		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);

		final TaskManagerLocation firstTaskManagerLocation = new LocalTaskManagerLocation();
		final TaskManagerLocation secondTaskManagerLocation = new LocalTaskManagerLocation();

		final List<AllocationID> firstTaskManagersSlots = registerAndOfferSlots(firstTaskManagerLocation, slotPool, 4);
		final List<AllocationID> secondTaskManagersSlots = registerAndOfferSlots(secondTaskManagerLocation, slotPool, 4);

		slotPool.allocateAvailableSlot(new SlotRequestId(), firstTaskManagersSlots.get(0));
		slotPool.allocateAvailableSlot(new SlotRequestId(), firstTaskManagersSlots.get(1));
		slotPool.allocateAvailableSlot(new SlotRequestId(), secondTaskManagersSlots.get(3));

		final Collection<SlotInfoWithUtilization> availableSlotsInformation = slotPool.getAvailableSlotsInformation();

		final Map<TaskManagerLocation, Double> utilizationPerTaskExecutor = ImmutableMap.of(
			firstTaskManagerLocation, 2.0 / 4,
			secondTaskManagerLocation, 1.0 / 4);

		for (SlotInfoWithUtilization slotInfoWithUtilization : availableSlotsInformation) {
			final double expectedTaskExecutorUtilization = utilizationPerTaskExecutor.get(slotInfoWithUtilization.getTaskManagerLocation());
			assertThat(slotInfoWithUtilization.getTaskExecutorUtilization(), is(closeTo(expectedTaskExecutorUtilization, 0.1)));
		}
	}
}
 
Example #12
Source File: PluginConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void getPluginsDir_nonExistingDirectory_returnsEmpty() {
	final Map<String, String> envVariables = ImmutableMap.of(ConfigConstants.ENV_FLINK_PLUGINS_DIR, new File(temporaryFolder.getRoot().getAbsoluteFile(), "should_not_exist").getAbsolutePath());
	CommonTestUtils.setEnv(envVariables);

	assertFalse(PluginConfig.getPluginsDir().isPresent());
}
 
Example #13
Source File: PluginConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void getPluginsDir_existingDirectory_returnsDirectoryFile() throws IOException {
	final File pluginsDirectory = temporaryFolder.newFolder();
	final Map<String, String> envVariables = ImmutableMap.of(ConfigConstants.ENV_FLINK_PLUGINS_DIR, pluginsDirectory.getAbsolutePath());
	CommonTestUtils.setEnv(envVariables);

	assertThat(PluginConfig.getPluginsDir().get(), is(pluginsDirectory));
}
 
Example #14
Source File: ParquetRecordReaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNestedMapGroup() throws IOException {
	Schema nestedMapSchema = unWrapSchema(NESTED_SCHEMA.getField("nestedMap").schema());
	Preconditions.checkState(nestedMapSchema.getType().equals(Schema.Type.MAP));

	Schema mapValueSchema = nestedMapSchema.getValueType();
	GenericRecord mapValue = new GenericRecordBuilder(mapValueSchema)
		.set("type", "nested")
		.set("value", "nested_value").build();

	ImmutableMap.Builder<String, GenericRecord> map = ImmutableMap.builder();
	map.put("testKey", mapValue);

	GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA)
		.set("nestedMap", map.build())
		.set("foo", 34L).build();

	Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record));
	MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA);
	ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema);

	InputFile inputFile =
		HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig);
	ParquetReadOptions options = ParquetReadOptions.builder().build();
	ParquetFileReader fileReader = new ParquetFileReader(inputFile, options);

	rowReader.initialize(fileReader, testConfig);
	assertFalse(rowReader.reachEnd());

	Row row = rowReader.nextRecord();
	assertEquals(7, row.getArity());

	assertEquals(34L, row.getField(0));
	Map result = (Map) row.getField(5);

	Row nestedRow = (Row) result.get("testKey");
	assertEquals("nested", nestedRow.getField(0));
	assertEquals("nested_value", nestedRow.getField(1));
}
 
Example #15
Source File: ParquetRecordReaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapGroup() throws IOException {
	Preconditions.checkState(unWrapSchema(NESTED_SCHEMA.getField("spamMap").schema())
		.getType().equals(Schema.Type.MAP));
	ImmutableMap.Builder<String, String> map = ImmutableMap.builder();
	map.put("testKey", "testValue");

	GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA)
		.set("foo", 32L)
		.set("spamMap", map.build())
		.build();

	Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record));
	MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA);
	ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema);

	InputFile inputFile =
		HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig);
	ParquetReadOptions options = ParquetReadOptions.builder().build();
	ParquetFileReader fileReader = new ParquetFileReader(inputFile, options);

	rowReader.initialize(fileReader, testConfig);
	assertFalse(rowReader.reachEnd());

	Row row = rowReader.nextRecord();
	assertEquals(7, row.getArity());

	assertEquals(32L, row.getField(0));
	Map<?, ?> result = (Map<?, ?>) row.getField(1);
	assertEquals(result.get("testKey").toString(), "testValue");
	assertTrue(rowReader.reachEnd());
}
 
Example #16
Source File: KafkaShuffleITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private Map<Integer, Collection<ConsumerRecord<byte[], byte[]>>> testKafkaShuffleProducer(
		String topic,
		StreamExecutionEnvironment env,
		int numberOfPartitions,
		int producerParallelism,
		int numElementsPerProducer,
		TimeCharacteristic timeCharacteristic) throws Exception {
	createTestTopic(topic, numberOfPartitions, 1);

	env.setParallelism(producerParallelism);
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.setStreamTimeCharacteristic(timeCharacteristic);

	DataStream<Tuple3<Integer, Long, Integer>> source =
		env.addSource(new KafkaSourceFunction(numElementsPerProducer, false)).setParallelism(producerParallelism);
	DataStream<Tuple3<Integer, Long, Integer>> input = (timeCharacteristic == EventTime) ?
		source.assignTimestampsAndWatermarks(new PunctuatedExtractor()).setParallelism(producerParallelism) : source;

	Properties properties = kafkaServer.getStandardProperties();
	Properties kafkaProperties = PropertiesUtil.flatten(properties);

	kafkaProperties.setProperty(PRODUCER_PARALLELISM, String.valueOf(producerParallelism));
	kafkaProperties.setProperty(PARTITION_NUMBER, String.valueOf(numberOfPartitions));
	kafkaProperties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
	kafkaProperties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
	FlinkKafkaShuffle.writeKeyBy(input, topic, kafkaProperties, 0);

	env.execute("Write to " + topic);
	ImmutableMap.Builder<Integer, Collection<ConsumerRecord<byte[], byte[]>>> results = ImmutableMap.builder();

	for (int p = 0; p < numberOfPartitions; p++) {
		results.put(p, kafkaServer.getAllRecordsFromTopic(kafkaProperties, topic, p, 5000));
	}

	deleteTestTopic(topic);

	return results.build();
}
 
Example #17
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassNoEntry() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of());

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertFalse(entry.isPresent());
}
 
Example #18
Source File: JarManifestParserTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassAssemblerClass() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("AssemblerClass")));
}
 
Example #19
Source File: JarManifestParserTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassMainClass() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("MainClass")));
}
 
Example #20
Source File: JarManifestParserTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassAssemblerClassAndMainClass() throws IOException {
	// We want the assembler class entry to have precedence over main class
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass",
		PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("AssemblerClass")));
}
 
Example #21
Source File: JarManifestParserTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws IOException {
	File jarWithNoManifest = createJarFileWithManifest(ImmutableMap.of());
	File jarFile = TestJob.getTestJobJar();

	JarFileWithEntryClass jarFileWithEntryClass = JarManifestParser
		.findOnlyEntryClass(ImmutableList.of(jarWithNoManifest, jarFile));

	assertThat(jarFileWithEntryClass.getEntryClass(), is(equalTo(TestJob.class.getCanonicalName())));
}
 
Example #22
Source File: ParquetRecordReaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapGroup() throws IOException {
	Preconditions.checkState(unWrapSchema(NESTED_SCHEMA.getField("spamMap").schema())
		.getType().equals(Schema.Type.MAP));
	ImmutableMap.Builder<String, String> map = ImmutableMap.builder();
	map.put("testKey", "testValue");

	GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA)
		.set("foo", 32L)
		.set("spamMap", map.build())
		.build();

	Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record));
	MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA);
	ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema);

	InputFile inputFile =
		HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig);
	ParquetReadOptions options = ParquetReadOptions.builder().build();
	ParquetFileReader fileReader = new ParquetFileReader(inputFile, options);

	rowReader.initialize(fileReader, testConfig);
	assertFalse(rowReader.reachEnd());

	Row row = rowReader.nextRecord();
	assertEquals(7, row.getArity());

	assertEquals(32L, row.getField(0));
	Map<?, ?> result = (Map<?, ?>) row.getField(1);
	assertEquals(result.get("testKey").toString(), "testValue");
	assertTrue(rowReader.reachEnd());
}
 
Example #23
Source File: ParquetRecordReaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNestedMapGroup() throws IOException {
	Schema nestedMapSchema = unWrapSchema(NESTED_SCHEMA.getField("nestedMap").schema());
	Preconditions.checkState(nestedMapSchema.getType().equals(Schema.Type.MAP));

	Schema mapValueSchema = nestedMapSchema.getValueType();
	GenericRecord mapValue = new GenericRecordBuilder(mapValueSchema)
		.set("type", "nested")
		.set("value", "nested_value").build();

	ImmutableMap.Builder<String, GenericRecord> map = ImmutableMap.builder();
	map.put("testKey", mapValue);

	GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA)
		.set("nestedMap", map.build())
		.set("foo", 34L).build();

	Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record));
	MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA);
	ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema);

	InputFile inputFile =
		HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig);
	ParquetReadOptions options = ParquetReadOptions.builder().build();
	ParquetFileReader fileReader = new ParquetFileReader(inputFile, options);

	rowReader.initialize(fileReader, testConfig);
	assertFalse(rowReader.reachEnd());

	Row row = rowReader.nextRecord();
	assertEquals(7, row.getArity());

	assertEquals(34L, row.getField(0));
	Map result = (Map) row.getField(5);

	Row nestedRow = (Row) result.get("testKey");
	assertEquals("nested", nestedRow.getField(0));
	assertEquals("nested_value", nestedRow.getField(1));
}
 
Example #24
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassAssemblerClass() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("AssemblerClass")));
}
 
Example #25
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassMainClass() throws IOException {
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("MainClass")));
}
 
Example #26
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindEntryClassAssemblerClassAndMainClass() throws IOException {
	// We want the assembler class entry to have precedence over main class
	File jarFile = createJarFileWithManifest(ImmutableMap.of(
		PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass",
		PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass"));

	Optional<String> entry = JarManifestParser.findEntryClass(jarFile);

	assertTrue(entry.isPresent());
	assertThat(entry.get(), is(equalTo("AssemblerClass")));
}
 
Example #27
Source File: JarManifestParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws IOException {
	File jarWithNoManifest = createJarFileWithManifest(ImmutableMap.of());
	File jarFile = TestJob.getTestJobJar();

	JarFileWithEntryClass jarFileWithEntryClass = JarManifestParser
		.findOnlyEntryClass(ImmutableList.of(jarWithNoManifest, jarFile));

	assertThat(jarFileWithEntryClass.getEntryClass(), is(equalTo(TestJob.class.getCanonicalName())));
}
 
Example #28
Source File: FlinkSqlParser.java    From sylph with Apache License 2.0 5 votes vote down vote up
private RealTimeTransForm getJoinTransForm(JoinContext joinContext, CreateTable batchTable)
{
    Map<String, Object> withConfig = batchTable.getWithConfig();
    String driverOrName = (String) withConfig.get("type");
    Class<?> driver = connectorStore.getConnectorDriver(driverOrName, Operator.PipelineType.transform);
    checkState(RealTimeTransForm.class.isAssignableFrom(driver), "batch table type driver must is RealTimeTransForm");

    // instance
    IocFactory iocFactory = IocFactory.create(binder -> binder.bind(JoinContext.class, joinContext));

    return NodeLoader.getPluginInstance(driver.asSubclass(RealTimeTransForm.class), iocFactory, ImmutableMap.copyOf(withConfig));
}
 
Example #29
Source File: JdbcAsyncJoin.java    From sylph with Apache License 2.0 4 votes vote down vote up
@Override
public void process(Record input, Collector<Record> collector)
{
    checkState(connection != null, " connection is null");

    StringBuilder builder = new StringBuilder();
    for (int index : joinOnMapping.keySet()) {
        builder.append(input.<Object>getField(index)).append("\u0001");
    }

    List<Map<String, Object>> cacheData = noCatch(() -> cache.get(builder.toString(), () -> {
        //-- 这里进行真正的数据库查询
        List<Integer> indexs = ImmutableList.copyOf(joinOnMapping.keySet());
        try (PreparedStatement statement = connection.prepareStatement(sql)) {
            for (int i = 0; i < indexs.size(); i++) {
                statement.setObject(i + 1, input.getField(indexs.get(i)));
            }
            if (logger.isDebugEnabled()) {
                logger.debug("Thread is  {}, this {}", Thread.currentThread().getId(), this);
            }
            try (ResultSet rs = statement.executeQuery()) {
                List<Map<String, Object>> result = JdbcUtils.resultToList(rs);
                if (result.isEmpty() && joinType == LEFT) { // left join and inter join
                    return ImmutableList.of(ImmutableMap.of());
                }
                return result;
            }
        }
        catch (SQLException e) {
            throw throwsException(e);
        }
    }));

    for (Map<String, Object> map : cacheData) {
        Object[] row = new Object[selectFieldCnt];
        for (int i = 0; i < selectFieldCnt; i++) {
            SelectField field = selectFields.get(i);
            if (field.isBatchTableField()) {
                row[i] = map.get(field.getFieldName());
            }
            else {
                row[i] = input.getField(field.getFieldIndex());
            }
        }
        collector.collect(Record.of(row));
    }
}
 
Example #30
Source File: LocalExecutorITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetSessionProperties() throws Exception {
	final Executor executor = createDefaultExecutor(clusterClient);

	final SessionContext session = new SessionContext("test-session", new Environment());
	session.getSessionEnv().setExecution(ImmutableMap.of("result-mode", "changelog"));
	// Open the session and get the sessionId.
	String sessionId = executor.openSession(session);
	try {
		assertEquals("test-session", sessionId);
		assertEquals(executor.getSessionProperties(sessionId).get("execution.result-mode"), "changelog");

		// modify defaults
		executor.setSessionProperty(sessionId, "execution.result-mode", "table");

		final Map<String, String> actualProperties = executor.getSessionProperties(sessionId);

		final Map<String, String> expectedProperties = new HashMap<>();
		expectedProperties.put("execution.planner", planner);
		expectedProperties.put("execution.type", "batch");
		expectedProperties.put("execution.time-characteristic", "event-time");
		expectedProperties.put("execution.periodic-watermarks-interval", "99");
		expectedProperties.put("execution.parallelism", "1");
		expectedProperties.put("execution.max-parallelism", "16");
		expectedProperties.put("execution.max-idle-state-retention", "600000");
		expectedProperties.put("execution.min-idle-state-retention", "1000");
		expectedProperties.put("execution.result-mode", "table");
		expectedProperties.put("execution.max-table-result-rows", "100");
		expectedProperties.put("execution.restart-strategy.type", "failure-rate");
		expectedProperties.put("execution.restart-strategy.max-failures-per-interval", "10");
		expectedProperties.put("execution.restart-strategy.failure-rate-interval", "99000");
		expectedProperties.put("execution.restart-strategy.delay", "1000");
		expectedProperties.put("table.optimizer.join-reorder-enabled", "false");
		expectedProperties.put("deployment.response-timeout", "5000");

		assertEquals(expectedProperties, actualProperties);

		// Reset session properties
		executor.resetSessionProperties(sessionId);
		assertEquals(executor.getSessionProperties(sessionId).get("execution.result-mode"), "changelog");
	} finally {
		executor.closeSession(sessionId);
	}
}