net.openhft.compiler.CompilerUtils Java Examples

The following examples show how to use net.openhft.compiler.CompilerUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Avro14FactoryTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testAddSchemaSupportToFixedClassNoNamespace() throws Exception {
  Schema parsed = AvroCompatibilityHelper.parse(FIXED_TYPE_NO_NAMESPACE_SCHEMA_JSON);
  Collection<AvroGeneratedSourceCode> compiled = _factory.compile(Collections.singletonList(parsed), AvroVersion.AVRO_1_4);
  Assert.assertEquals(1, compiled.size());
  AvroGeneratedSourceCode sourceCode = compiled.iterator().next();

  Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava("Whatever", sourceCode.getContents());
  Assert.assertNotNull(aClass);
  Assert.assertTrue(SpecificFixed.class.isAssignableFrom(aClass));
  Field schemaField = aClass.getField("SCHEMA$");
  Assert.assertNotNull(schemaField);
  Schema schema = (Schema) schemaField.get(null);
  Assert.assertNotNull(schema);
  Assert.assertEquals("Whatever", schema.getName());
  Assert.assertNull(schema.getNamespace());
  Assert.assertEquals(42, schema.getFixedSize());
  Assert.assertTrue(schema.getDoc().contains("w00t"));
}
 
Example #2
Source File: Avro14FactoryTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testAddSchemaSupportToFixedClass() throws Exception {
  Schema parsed = AvroCompatibilityHelper.parse(FIXED_TYPE_SCHEMA_JSON);
  Collection<AvroGeneratedSourceCode> compiled = _factory.compile(Collections.singletonList(parsed), AvroVersion.AVRO_1_4);
  Assert.assertEquals(1, compiled.size());
  AvroGeneratedSourceCode sourceCode = compiled.iterator().next();

  Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava("com.acme.Whatever", sourceCode.getContents());
  Assert.assertNotNull(aClass);
  Assert.assertTrue(SpecificFixed.class.isAssignableFrom(aClass));
  Field schemaField = aClass.getField("SCHEMA$");
  Assert.assertNotNull(schemaField);
  Schema schema = (Schema) schemaField.get(null);
  Assert.assertNotNull(schema);
  Assert.assertEquals("Whatever", schema.getName());
  Assert.assertEquals("com.acme", schema.getNamespace());
  Assert.assertEquals(42, schema.getFixedSize());
  Assert.assertTrue(schema.getDoc().contains("yadda yadda"));
}
 
Example #3
Source File: Avro14FactoryTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testAddSchemaSupportToEnumNoNamespace() throws Exception {
  Schema parsed = AvroCompatibilityHelper.parse(ENUM_CLASS_NO_NAMESPACE_JSON);
  Collection<AvroGeneratedSourceCode> compiled = _factory.compile(Collections.singletonList(parsed), AvroVersion.AVRO_1_4);
  Assert.assertEquals(1, compiled.size());
  AvroGeneratedSourceCode sourceCode = compiled.iterator().next();

  Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava("BobSmith", sourceCode.getContents());
  Assert.assertNotNull(aClass);
  Field schemaField = aClass.getField("SCHEMA$");
  Assert.assertNotNull(schemaField);
  Schema schema = (Schema) schemaField.get(null);
  Assert.assertNotNull(schema);
  Assert.assertEquals("BobSmith", schema.getName());
  Assert.assertNull(schema.getNamespace());
  Assert.assertTrue(schema.getDoc().contains("Bob Smith Store"));
}
 
Example #4
Source File: Avro14FactoryTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testAddSchemaSupportToEnum() throws Exception {
  Schema parsed = AvroCompatibilityHelper.parse(ENUM_CLASS_JSON);
  Collection<AvroGeneratedSourceCode> compiled = _factory.compile(Collections.singletonList(parsed), AvroVersion.AVRO_1_4);
  Assert.assertEquals(1, compiled.size());
  AvroGeneratedSourceCode sourceCode = compiled.iterator().next();

  Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava("com.dot.BobSmith", sourceCode.getContents());
  Assert.assertNotNull(aClass);
  Field schemaField = aClass.getField("SCHEMA$");
  Assert.assertNotNull(schemaField);
  Schema schema = (Schema) schemaField.get(null);
  Assert.assertNotNull(schema);
  Assert.assertEquals("BobSmith", schema.getName());
  Assert.assertEquals("com.dot", schema.getNamespace());
  Assert.assertTrue(schema.getDoc().contains("Bob Smith Store"));
}
 
Example #5
Source File: Avro14FactoryCompatibilityTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testVanilla14FixedClassesIncompatibleWithModernAvro() throws Exception {
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (!runtimeVersion.laterThan(AvroVersion.AVRO_1_7)) {
    throw new SkipException("class only supported under modern avro. runtime version detected as " + runtimeVersion);
  }

  String sourceCode = TestUtil.load("Vanilla14Fixed");
  StringWriter sr = new StringWriter();
  PrintWriter compilerOutput = new PrintWriter(sr);
  try {
    Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(getClass().getClassLoader(),
        "com.acme.generatedby14.Vanilla14Fixed", sourceCode, compilerOutput);
    Assert.fail("compilation expected to fail");
  } catch (ClassNotFoundException ignored) {
    //expected
  }
  String errorMsg = sr.toString();
  Assert.assertTrue(errorMsg.contains("is not abstract and does not override")); //doesnt implement Externalizable
}
 
Example #6
Source File: Avro14FactoryCompatibilityTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testVanilla14FixedClassesIncompatibleWithAvro17() throws Exception {
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (!runtimeVersion.equals(AvroVersion.AVRO_1_7)) {
    throw new SkipException("class only supported under avro 1.7. runtime version detected as " + runtimeVersion);
  }

  String sourceCode = TestUtil.load("Vanilla14Fixed");
  Class clazz = CompilerUtils.CACHED_COMPILER.loadFromJava("com.acme.generatedby14.Vanilla14Fixed", sourceCode);
  try {
    clazz.newInstance();
    Assert.fail("expecting an exception");
  } catch (AvroRuntimeException expected) {
    Assert.assertTrue(expected.getMessage().contains("Not a Specific class")); //fails to find SCHEMA$
  }
}
 
Example #7
Source File: CodeTransformationsAvro14Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro14HugeRecord() throws Exception {
  String avsc = TestUtil.load("HugeRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_4, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #8
Source File: CodeBuilder.java    From cst with GNU Lesser General Public License v3.0 5 votes vote down vote up
public static Class compile(String className, String code) {
    Class aClass = null;

    try {
        aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, code);
    } catch (Exception e) {
        e.printStackTrace();
    }

    return aClass;
}
 
Example #9
Source File: CodeGenFlinkTable.java    From df_data_service with Apache License 2.0 5 votes vote down vote up
public static Table getFlinkTableObj(String className, String javaCode){
	try {
		Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
		DynamicRunner runner = (DynamicRunner) aClass.newInstance();
		//return runner.getTableObj();

	} catch (Exception e) {
		e.printStackTrace();
	}
	return null;
}
 
Example #10
Source File: HandlerGenerator.java    From proteus with Apache License 2.0 5 votes vote down vote up
/**
 * Compiles the generated source into a new {@link Class}
 * @return a new {@code Supplier<RoutingHandler>} class
 */
public Class<? extends Supplier<RoutingHandler>> compileClass()
{
    try {
        this.generateRoutes();

        log.debug("\n\nGenerated Class Source:\n\n" + this.sourceString);

        return CompilerUtils.CACHED_COMPILER.loadFromJava(packageName + "." + className, this.sourceString);

    } catch (Exception e) {
        log.error(e.getMessage(), e);
        return null;
    }
}
 
Example #11
Source File: CodeTransformationsAvro17Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro17HugeRecord() throws Exception {
  String avsc = TestUtil.load("HugeRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_7, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #12
Source File: CodeTransformationsAvro17Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro17Enum() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalEnum.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformEnumClass(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
  Assert.assertTrue(Enum.class.isAssignableFrom(transformedClass));
}
 
Example #13
Source File: CodeTransformationsAvro14Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro14RecordWithMultilineDoc() throws Exception {
  String avsc = TestUtil.load("RecordWithMultilineDoc.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_4, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #14
Source File: CodeTransformationsAvro14Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro14Enum() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalEnum.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformEnumClass(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
  Assert.assertTrue(Enum.class.isAssignableFrom(transformedClass));
}
 
Example #15
Source File: CodeTransformationsAvro18Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro18Enum() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalEnum.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformEnumClass(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
  Assert.assertTrue(Enum.class.isAssignableFrom(transformedClass));
}
 
Example #16
Source File: CodeTransformationsAvro19Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro19Enum() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalEnum.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformEnumClass(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
  Assert.assertTrue(Enum.class.isAssignableFrom(transformedClass));
}
 
Example #17
Source File: CodeTransformationsAvro19Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro19parseCalls() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_9, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #18
Source File: CodeTransformationsAvro19Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro19HugeRecord() throws Exception {
  String avsc = TestUtil.load("HugeRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_9, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #19
Source File: CodeTransformationsAvro19Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testRemoveAvro19Builder() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.removeBuilderSupport(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #20
Source File: CodeTransformationsAvro19Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro19Externalizable() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformExternalizableSupport(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #21
Source File: CodeTransformationsAvro18Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro18HugeRecord() throws Exception {
  String avsc = TestUtil.load("HugeRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_8, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #22
Source File: CodeTransformationsAvro15Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro15Enum() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalEnum.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);

  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformEnumClass(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
  Assert.assertTrue(Enum.class.isAssignableFrom(transformedClass));
}
 
Example #23
Source File: CodeTransformationsAvro15Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro15HugeRecord() throws Exception {
  String avsc = TestUtil.load("HugeRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_5, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #24
Source File: CodeTransformationsAvro15Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro15RecordWithMultilineDoc() throws Exception {
  String avsc = TestUtil.load("RecordWithMultilineDoc.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_5, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #25
Source File: CodeTransformationsAvro16Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro16Enum() throws Exception {
  String avsc = TestUtil.load("PerfectlyNormalEnum.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformEnumClass(originalCode, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
  Assert.assertTrue(Enum.class.isAssignableFrom(transformedClass));
}
 
Example #26
Source File: CodeTransformationsAvro16Test.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testTransformAvro16HugeRecord() throws Exception {
  String avsc = TestUtil.load("HugeRecord.avsc");
  Schema schema = AvroCompatibilityHelper.parse(avsc);
  String originalCode = runNativeCodegen(schema);

  String transformedCode = CodeTransformations.transformParseCalls(originalCode, AvroVersion.AVRO_1_6, AvroVersion.earliest(), AvroVersion.latest());

  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava(schema.getFullName(), transformedCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #27
Source File: Avro14FutureGeneratedCodeAtCompileTest.java    From avro-util with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Test
public void demonstrateAvro19CompatibleCode() throws Exception {
  String sourceCode = TestUtil.load("under19/SimpleRecord.java");
  Class<?> transformedClass = CompilerUtils.CACHED_COMPILER.loadFromJava("under19.SimpleRecord", sourceCode);
  Assert.assertNotNull(transformedClass);
}
 
Example #28
Source File: FlinkAvroTableAPIClient.java    From df_data_service with Apache License 2.0 4 votes vote down vote up
public static void tcFlinkAvroTableAPI(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String transScript) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopic, new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result;
        Table ingest = tableEnv.scan(srcTopic);
        String className = "dynamic.FlinkScript";
        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";
        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl." + transScript + ";\n" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";
        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        result = runner.transTableObj(ingest);

        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkTableAPI_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #29
Source File: CodeGenFlinkTable.java    From df_data_service with Apache License 2.0 4 votes vote down vote up
public static void main(String args[]) {

		String transform = "flatMap(new FlinkUDF.LineSplitter()).groupBy(0).sum(1).print();\n";

		String transform2 = "select(\"name\");\n";

		String header = "package dynamic;\n" +
				"import org.apache.flink.api.table.Table;\n" +
				"import com.datafibers.util.*;\n";

		String javaCode = header +
				"public class FlinkScript implements DynamicRunner {\n" +
				"@Override \n" +
				"    public void runTransform(DataSet<String> ds) {\n" +
						"try {" +
						"ds."+ transform +
						"} catch (Exception e) {" +
						"};" +
				"}}";

		String javaCode2 = header +
				"public class FlinkScript implements DynamicRunner {\n" +
				"@Override \n" +
				"    public Table transTableObj(Table tbl) {\n" +
					"try {" +
					"return tbl."+ transform2 +
					"} catch (Exception e) {" +
					"};" +
					"return null;}}";

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
		CsvTableSource csvTableSource = new CsvTableSource(
				"/Users/will/Downloads/file.csv",
				new String[] { "name", "id", "score", "comments" },
				new TypeInformation<?>[] {
						Types.STRING(),
						Types.STRING(),
						Types.STRING(),
						Types.STRING()
				}); // lenient

		tableEnv.registerTableSource("mycsv", csvTableSource);
		TableSink sink = new CsvTableSink("/Users/will/Downloads/out.csv", "|");
		Table ingest = tableEnv.scan("mycsv");

		try {
			String className = "dynamic.FlinkScript";
			Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode2);
			DynamicRunner runner = (DynamicRunner) aClass.newInstance();
			//runner.runTransform(ds);
			Table result = runner.transTableObj(ingest);
			// write the result Table to the TableSink
			result.writeToSink(sink);
			env.execute();

		} catch (Exception e) {
			e.printStackTrace();
		}
	}
 
Example #30
Source File: UnitTestSuiteFlink.java    From df_data_service with Apache License 2.0 4 votes vote down vote up
public static void testFlinkAvroScriptWithStaticSchema() {
    System.out.println("TestCase_Test Avro Table API Script with static Schema");

    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("static.avro.schema", STATIC_USER_SCHEMA);

    try {
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table ingest = tableEnv.scan("Orders");

        String className = "dynamic.FlinkScript";

        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";

        String transScript = "select(\"name\")";

        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl."+ transScript + ";" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";

        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        Table result = runner.transTableObj(ingest);

        Kafka09AvroTableSink sink =
                new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());
        // write the result Table to the TableSink
        result.writeToSink(sink);
        env.execute("Flink AVRO SQL KAFKA Test");
    } catch (Exception e) {
        e.printStackTrace();
    }
}