Java Code Examples for org.apache.flink.util.FileUtils#readFileUtf8()

The following examples show how to use org.apache.flink.util.FileUtils#readFileUtf8() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JsonRowSchemaConverterTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}

Example 2

Source File: SQLClientHBaseITCase.java From flink with Apache License 2.0

6 votes

@Before
public void before() throws Exception {
	DOWNLOAD_CACHE.before();
	Path tmpPath = tmp.getRoot().toPath();
	LOG.info("The current temporary path: {}", tmpPath);

	// Prepare all hadoop jars to mock HADOOP_CLASSPATH, use hadoop.classpath which contains all hadoop jars
	File hadoopClasspathFile = new File(hadoopClasspath.toAbsolutePath().toString());

	if (!hadoopClasspathFile.exists()) {
		throw new FileNotFoundException("File that contains hadoop classpath " + hadoopClasspath.toString()
			+ " does not exist.");
	}

	String classPathContent = FileUtils.readFileUtf8(hadoopClasspathFile);
	hadoopClasspathJars = Arrays.stream(classPathContent.split(":"))
		.map(jar -> Paths.get(jar))
		.collect(Collectors.toList());
}

Example 3

Source File: FileCacheDirectoriesTest.java From flink with Apache License 2.0

6 votes

private void testDirectoryDownloaded(DistributedCache.DistributedCacheEntry entry) throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	// copy / create the file
	final String fileName = "test_file";
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}

Example 4

Source File: JsonRowSchemaConverterTest.java From flink with Apache License 2.0

6 votes

@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}

Example 5

Source File: FileCacheDirectoriesTest.java From flink with Apache License 2.0

6 votes

@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}

Example 6

Source File: JsonRowSchemaConverterTest.java From flink with Apache License 2.0

6 votes

@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}

Example 7

Source File: FileCacheDirectoriesTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}

Example 8

Source File: EnvironmentFileUtil.java From pulsar-flink with Apache License 2.0

5 votes

public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
    final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
    Objects.requireNonNull(url);
    String schema = FileUtils.readFileUtf8(new File(url.getFile()));

    for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
        schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
    }

    return Environment.parse(schema);
}

Example 9

Source File: EnvironmentFileUtil.java From flink with Apache License 2.0

5 votes

public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
	final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
	Objects.requireNonNull(url);
	String schema = FileUtils.readFileUtf8(new File(url.getFile()));

	for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
		schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
	}

	return Environment.parse(schema);
}

Example 10

Source File: JsonRowSchemaConverterTest.java From flink with Apache License 2.0

5 votes

@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}

Example 11

Source File: EnvironmentFileUtil.java From flink with Apache License 2.0

5 votes

public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
	final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
	Objects.requireNonNull(url);
	String schema = FileUtils.readFileUtf8(new File(url.getFile()));

	for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
		schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
	}

	return Environment.parse(schema);
}

Example 12

Source File: JsonRowSchemaConverterTest.java From flink with Apache License 2.0

5 votes

@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}

Example 13

Source File: JsonRowSchemaConverterTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}

Example 14

Source File: EnvironmentFileUtil.java From Flink-CEPplus with Apache License 2.0

5 votes

public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
	final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
	Objects.requireNonNull(url);
	String schema = FileUtils.readFileUtf8(new File(url.getFile()));

	for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
		schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
	}

	return Environment.parse(schema);
}

Example 15

Source File: TupleGenerator.java From flink with Apache License 2.0

4 votes

private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		s.close();
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}

Example 16

Source File: TupleGenerator.java From flink with Apache License 2.0

4 votes

private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}

Example 17

Source File: TupleGenerator.java From Flink-CEPplus with Apache License 2.0

4 votes

private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}

Example 18

Source File: TupleGenerator.java From Flink-CEPplus with Apache License 2.0

4 votes

private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		s.close();
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}

Example 19

Source File: TupleGenerator.java From flink with Apache License 2.0

4 votes

private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		s.close();
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}

Example 20

Source File: TupleGenerator.java From flink with Apache License 2.0

4 votes

private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}