Java Code Examples for org.apache.flink.api.common.io.FileOutputFormat

The following examples show how to use org.apache.flink.api.common.io.FileOutputFormat. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: CsvOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 
Example 2
Source Project: Flink-CEPplus   Source File: CsvOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 
Example 3
Source Project: flink   Source File: CsvOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 
Example 4
Source Project: flink   Source File: CsvOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 
Example 5
Source Project: flink   Source File: CsvOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 
Example 6
Source Project: flink   Source File: CsvOutputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: HDFSTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 8
Source Project: Flink-CEPplus   Source File: TaskTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(FileOutputFormat<Record> outputFormat, String outPath) {
	TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	dsConfig.setStubWrapper(new UserCodeObjectWrapper<>(outputFormat));
}
 
Example 9
Source Project: flink   Source File: HDFSTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 10
Source Project: flink   Source File: TaskTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	Class<? extends FileOutputFormat<Record>> stubClass,
	String outPath,
	Configuration formatParams) {

	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath, formatParams);
}
 
Example 11
Source Project: flink   Source File: TaskTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 
Example 12
Source Project: incubator-iotdb   Source File: TsFileOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
protected Path getAcutalFilePath() {
	try {
		Field field = FileOutputFormat.class.getDeclaredField("actualFilePath");
		field.setAccessible(true);
		return (Path) field.get(this);
	} catch (NoSuchFieldException | IllegalAccessException e) {
		throw new RuntimeException("Get actual file path failed!", e);
	}
}
 
Example 13
Source Project: rheem   Source File: RheemFileOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
public void setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode mode) {
    if (mode == null) {
        throw new NullPointerException();
    }

    this.outputDirectoryMode = mode;
}
 
Example 14
Source Project: rheem   Source File: RheemFileOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
    try {
        final Path path = getOutputFilePath();
        final FileSystem fs = path.getFileSystem();

        // only distributed file systems can be initialized at start-up time.
        if (fs.isDistributedFS()) {

            final FileSystem.WriteMode writeMode = getWriteMode();
            final FileOutputFormat.OutputDirectoryMode outDirMode = getOutputDirectoryMode();

            if (parallelism == 1 && outDirMode == FileOutputFormat.OutputDirectoryMode.PARONLY) {
                // output is not written in parallel and should be written to a single file.
                // prepare distributed output path
                if (!fs.initOutPathDistFS(path, writeMode, false)) {
                    // output preparation failed! Cancel task.
                    throw new IOException("Output path could not be initialized.");
                }

            } else {
                // output should be written to a directory

                // only distributed file systems can be initialized at start-up time.
                if (!fs.initOutPathDistFS(path, writeMode, true)) {
                    throw new IOException("Output directory could not be created.");
                }
            }
        }
    }catch (Exception e){
        throw new RheemException(e);
    }
}
 
Example 15
Source Project: flink   Source File: ScalaCsvOutputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	path = tmpFolder.newFile().getAbsolutePath();
	csvOutputFormat = new ScalaCsvOutputFormat<>(new Path(path));
	csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
	csvOutputFormat.open(0, 1);
}
 
Example 16
Source Project: flink   Source File: HDFSTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 17
Source Project: flink   Source File: TaskTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	Class<? extends FileOutputFormat<Record>> stubClass,
	String outPath,
	Configuration formatParams) {

	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath, formatParams);
}
 
Example 18
Source Project: flink   Source File: TaskTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 
Example 19
Source Project: Flink-CEPplus   Source File: MiniCluster.java    License: Apache License 2.0 4 votes vote down vote up
private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 
Example 20
Source Project: Flink-CEPplus   Source File: TaskTestBase.java    License: Apache License 2.0 4 votes vote down vote up
public void registerFileOutputTask(Class<? extends FileOutputFormat<Record>> stubClass, String outPath) {
	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath);
}
 
Example 21
Source Project: flink   Source File: MiniCluster.java    License: Apache License 2.0 4 votes vote down vote up
private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 
Example 22
Source Project: rheem   Source File: RheemFileOutputFormat.java    License: Apache License 2.0 4 votes vote down vote up
public FileOutputFormat.OutputDirectoryMode getOutputDirectoryMode() {
    return this.outputDirectoryMode;
}
 
Example 23
Source Project: flink   Source File: MiniCluster.java    License: Apache License 2.0 4 votes vote down vote up
private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 
Example 24
Source Project: Flink-CEPplus   Source File: DataSet.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 
Example 25
Source Project: Flink-CEPplus   Source File: DataSet.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}
 
Example 26
Source Project: flink   Source File: DataSet.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 
Example 27
Source Project: flink   Source File: DataSet.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}
 
Example 28
Source Project: rheem   Source File: RheemFileOutputFormat.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Initialize defaults for output format. Needs to be a static method because it is configured for local
 * cluster execution, see LocalFlinkMiniCluster.
 * @param configuration The configuration to load defaults from
 */
public static void initDefaultsFromConfiguration(Configuration configuration) {

    DEFAULT_WRITE_MODE = FileSystem.WriteMode.OVERWRITE ;


    DEFAULT_OUTPUT_DIRECTORY_MODE =  FileOutputFormat.OutputDirectoryMode.PARONLY;
}
 
Example 29
Source Project: flink   Source File: DataSet.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 
Example 30
Source Project: flink   Source File: DataSet.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}