org.apache.flink.api.common.io.FileOutputFormat Java Examples

The following examples show how to use org.apache.flink.api.common.io.FileOutputFormat. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CsvOutputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 
Example #2
Source File: CsvOutputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 
Example #3
Source File: CsvOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 
Example #4
Source File: CsvOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 
Example #5
Source File: CsvOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 
Example #6
Source File: CsvOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 
Example #7
Source File: TaskTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 
Example #8
Source File: TaskTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 
Example #9
Source File: TaskTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	Class<? extends FileOutputFormat<Record>> stubClass,
	String outPath,
	Configuration formatParams) {

	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath, formatParams);
}
 
Example #10
Source File: HDFSTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #11
Source File: ScalaCsvOutputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	path = tmpFolder.newFile().getAbsolutePath();
	csvOutputFormat = new ScalaCsvOutputFormat<>(new Path(path));
	csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
	csvOutputFormat.open(0, 1);
}
 
Example #12
Source File: RheemFileOutputFormat.java    From rheem with Apache License 2.0 5 votes vote down vote up
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
    try {
        final Path path = getOutputFilePath();
        final FileSystem fs = path.getFileSystem();

        // only distributed file systems can be initialized at start-up time.
        if (fs.isDistributedFS()) {

            final FileSystem.WriteMode writeMode = getWriteMode();
            final FileOutputFormat.OutputDirectoryMode outDirMode = getOutputDirectoryMode();

            if (parallelism == 1 && outDirMode == FileOutputFormat.OutputDirectoryMode.PARONLY) {
                // output is not written in parallel and should be written to a single file.
                // prepare distributed output path
                if (!fs.initOutPathDistFS(path, writeMode, false)) {
                    // output preparation failed! Cancel task.
                    throw new IOException("Output path could not be initialized.");
                }

            } else {
                // output should be written to a directory

                // only distributed file systems can be initialized at start-up time.
                if (!fs.initOutPathDistFS(path, writeMode, true)) {
                    throw new IOException("Output directory could not be created.");
                }
            }
        }
    }catch (Exception e){
        throw new RheemException(e);
    }
}
 
Example #13
Source File: RheemFileOutputFormat.java    From rheem with Apache License 2.0 5 votes vote down vote up
public void setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode mode) {
    if (mode == null) {
        throw new NullPointerException();
    }

    this.outputDirectoryMode = mode;
}
 
Example #14
Source File: TsFileOutputFormat.java    From incubator-iotdb with Apache License 2.0 5 votes vote down vote up
protected Path getAcutalFilePath() {
	try {
		Field field = FileOutputFormat.class.getDeclaredField("actualFilePath");
		field.setAccessible(true);
		return (Path) field.get(this);
	} catch (NoSuchFieldException | IllegalAccessException e) {
		throw new RuntimeException("Get actual file path failed!", e);
	}
}
 
Example #15
Source File: TaskTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	Class<? extends FileOutputFormat<Record>> stubClass,
	String outPath,
	Configuration formatParams) {

	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath, formatParams);
}
 
Example #16
Source File: HDFSTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #17
Source File: TaskTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(FileOutputFormat<Record> outputFormat, String outPath) {
	TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	dsConfig.setStubWrapper(new UserCodeObjectWrapper<>(outputFormat));
}
 
Example #18
Source File: HDFSTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #19
Source File: MiniCluster.java    From flink with Apache License 2.0 4 votes vote down vote up
private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 
Example #20
Source File: RheemFileOutputFormat.java    From rheem with Apache License 2.0 4 votes vote down vote up
public FileOutputFormat.OutputDirectoryMode getOutputDirectoryMode() {
    return this.outputDirectoryMode;
}
 
Example #21
Source File: TaskTestBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public void registerFileOutputTask(Class<? extends FileOutputFormat<Record>> stubClass, String outPath) {
	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath);
}
 
Example #22
Source File: MiniCluster.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 
Example #23
Source File: MiniCluster.java    From flink with Apache License 2.0 4 votes vote down vote up
private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 
Example #24
Source File: DataSet.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 
Example #25
Source File: RheemFileOutputFormat.java    From rheem with Apache License 2.0 3 votes vote down vote up
/**
 * Initialize defaults for output format. Needs to be a static method because it is configured for local
 * cluster execution, see LocalFlinkMiniCluster.
 * @param configuration The configuration to load defaults from
 */
public static void initDefaultsFromConfiguration(Configuration configuration) {

    DEFAULT_WRITE_MODE = FileSystem.WriteMode.OVERWRITE ;


    DEFAULT_OUTPUT_DIRECTORY_MODE =  FileOutputFormat.OutputDirectoryMode.PARONLY;
}
 
Example #26
Source File: DataSet.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}
 
Example #27
Source File: DataSet.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 
Example #28
Source File: DataSet.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 
Example #29
Source File: DataSet.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}
 
Example #30
Source File: DataSet.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up
/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}