Java Code Examples for org.apache.flink.core.fs.Path

The following examples show how to use org.apache.flink.core.fs.Path. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: OrcRowInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an OrcRowInputFormat.
 *
 * @param path The path to read ORC files from.
 * @param orcSchema The schema of the ORC files as ORC TypeDescription.
 * @param orcConfig The configuration to read the ORC files with.
 * @param batchSize The number of Row objects to read in a batch.
 */
public OrcRowInputFormat(String path, TypeDescription orcSchema, Configuration orcConfig, int batchSize) {
	super(new Path(path));

	// configure OrcRowInputFormat
	this.schema = orcSchema;
	this.rowType = (RowTypeInfo) OrcBatchReader.schemaToTypeInfo(schema);
	this.conf = orcConfig;
	this.batchSize = batchSize;

	// set default selection mask, i.e., all fields.
	this.selectedFields = new int[this.schema.getChildren().size()];
	for (int i = 0; i < selectedFields.length; i++) {
		this.selectedFields[i] = i;
	}
}
 
Example 3
Source Project: flink   Source File: HadoopPathBasedBulkFormatBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public HadoopPathBasedBulkFormatBuilder(
	org.apache.hadoop.fs.Path basePath,
	HadoopPathBasedBulkWriter.Factory<IN> writerFactory,
	HadoopFileCommitterFactory fileCommitterFactory,
	Configuration configuration,
	BucketAssigner<IN, BucketID> assigner,
	CheckpointRollingPolicy<IN, BucketID> policy,
	BucketFactory<IN, BucketID> bucketFactory,
	OutputFileConfig outputFileConfig) {

	this.basePath = new Path(Preconditions.checkNotNull(basePath).toString());
	this.writerFactory = writerFactory;
	this.fileCommitterFactory = fileCommitterFactory;
	this.serializableConfiguration = new SerializableConfiguration(configuration);
	this.bucketAssigner = Preconditions.checkNotNull(assigner);
	this.rollingPolicy = Preconditions.checkNotNull(policy);
	this.bucketFactory = Preconditions.checkNotNull(bucketFactory);
	this.outputFileConfig = Preconditions.checkNotNull(outputFileConfig);
}
 
Example 4
private void verifyPythonDriverOptionsParsing(final String[] args) throws FlinkParseException {
	final PythonDriverOptions pythonCommandOptions = commandLineParser.parse(args);

	// verify the parsed python entrypoint module
	assertEquals("xxx", pythonCommandOptions.getEntrypointModule());

	// verify the parsed python library files
	final List<Path> pythonMainFile = pythonCommandOptions.getPythonLibFiles();
	assertNotNull(pythonMainFile);
	assertEquals(4, pythonMainFile.size());
	assertEquals(
		pythonMainFile.stream().map(Path::getName).collect(Collectors.joining(",")),
		"xxx.py,a.py,b.py,c.py");

	// verify the python program arguments
	final List<String> programArgs = pythonCommandOptions.getProgramArgs();
	assertEquals(2, programArgs.size());
	assertEquals("--input", programArgs.get(0));
	assertEquals("in.txt", programArgs.get(1));
}
 
Example 5
Source Project: Flink-CEPplus   Source File: HadoopSwiftFileSystemITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	Assume.assumeTrue("Swift container not configured, skipping test...", CONTAINER != null);
	Assume.assumeTrue("Swift username not configured, skipping test...", USERNAME != null);
	Assume.assumeTrue("Swift password not configured, skipping test...", PASSWORD != null);
	Assume.assumeTrue("Swift tenant not configured, skipping test...", TENANT != null);
	Assume.assumeTrue("Swift region not configured, skipping test...", REGION != null);

	// initialize configuration with valid credentials
	final Configuration conf = createConfiguration();

	FileSystem.initialize(conf);

	// check for uniqueness of the test directory
	final Path directory = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR);
	final FileSystem fs = directory.getFileSystem();

	// directory must not yet exist
	assertFalse(fs.exists(directory));

	// reset configuration
	FileSystem.initialize(new Configuration());

	skipTest = false;
}
 
Example 6
Source Project: flink   Source File: LocalFileSystem.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(final Path f) throws IOException {

	final File localf = pathToFile(f);
	FileStatus[] results;

	if (!localf.exists()) {
		return null;
	}
	if (localf.isFile()) {
		return new FileStatus[] { new LocalFileStatus(localf, this) };
	}

	final String[] names = localf.list();
	if (names == null) {
		return null;
	}
	results = new FileStatus[names.length];
	for (int i = 0; i < names.length; i++) {
		results[i] = getFileStatus(new Path(f, names[i]));
	}

	return results;
}
 
Example 7
Source Project: flink   Source File: CheckpointStateOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 
Example 8
Source Project: flink   Source File: TestEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	OptimizedPlan op = compileProgram(jobName);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jobGraph = jgg.compileJobGraph(op);

	for (Path jarFile: jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph);
	return this.lastJobExecutionResult;
}
 
Example 9
Source Project: Flink-CEPplus   Source File: DefaultBucketFactoryImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Bucket<IN, BucketID> getNewBucket(
		final RecoverableWriter fsWriter,
		final int subtaskIndex,
		final BucketID bucketId,
		final Path bucketPath,
		final long initialPartCounter,
		final PartFileWriter.PartFileFactory<IN, BucketID> partFileWriterFactory,
		final RollingPolicy<IN, BucketID> rollingPolicy) {

	return Bucket.getNew(
			fsWriter,
			subtaskIndex,
			bucketId,
			bucketPath,
			initialPartCounter,
			partFileWriterFactory,
			rollingPolicy);
}
 
Example 10
Source Project: flink   Source File: ContinuousFileProcessingITCase.java    License: Apache License 2.0 6 votes vote down vote up
/** Create a file and fill it with content. */
private Tuple2<org.apache.hadoop.fs.Path, String> fillWithData(
	String base, String fileName, int fileIdx, String sampleLine) throws IOException, InterruptedException {

	assert (hdfs != null);

	org.apache.hadoop.fs.Path tmp =
		new org.apache.hadoop.fs.Path(base + "/." + fileName + fileIdx);

	FSDataOutputStream stream = hdfs.create(tmp);
	StringBuilder str = new StringBuilder();
	for (int i = 0; i < LINES_PER_FILE; i++) {
		String line = fileIdx + ": " + sampleLine + " " + i + "\n";
		str.append(line);
		stream.write(line.getBytes(ConfigConstants.DEFAULT_CHARSET));
	}
	stream.close();
	return new Tuple2<>(tmp, str.toString());
}
 
Example 11
Source Project: flink   Source File: FileCacheReadsFromBlobTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}
 
Example 12
Source Project: Flink-CEPplus   Source File: OuterJoinOperatorBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 
Example 13
Source Project: flink   Source File: ParquetPojoInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadPojoFromSimpleRecord() throws IOException {
	Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> simple = TestUtil.getSimpleRecordTestData();
	MessageType messageType = SCHEMA_CONVERTER.convert(TestUtil.SIMPLE_SCHEMA);
	Path path = TestUtil.createTempParquetFile(tempRoot.getRoot(), TestUtil.SIMPLE_SCHEMA, Collections.singletonList(simple.f1));

	ParquetPojoInputFormat<PojoSimpleRecord> inputFormat = new ParquetPojoInputFormat<>(
		path, messageType, (PojoTypeInfo<PojoSimpleRecord>) Types.POJO(PojoSimpleRecord.class));
	inputFormat.setRuntimeContext(TestUtil.getMockRuntimeContext());

	FileInputSplit[] splits = inputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	inputFormat.open(splits[0]);

	PojoSimpleRecord simpleRecord = inputFormat.nextRecord(null);
	assertEquals(simple.f2.getField(0), simpleRecord.getFoo());
	assertEquals(simple.f2.getField(1), simpleRecord.getBar());
	assertArrayEquals((Long[]) simple.f2.getField(2), simpleRecord.getArr());
}
 
Example 14
Source Project: Flink-CEPplus   Source File: BlobClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Uploads the JAR files to the {@link PermanentBlobService} of the {@link BlobServer} at the
 * given address with HA as configured.
 *
 * @param serverAddress
 * 		Server address of the {@link BlobServer}
 * @param clientConfig
 * 		Any additional configuration for the blob client
 * @param jobId
 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
 * @param files
 * 		List of files to upload
 *
 * @throws IOException
 * 		if the upload fails
 */
public static List<PermanentBlobKey> uploadFiles(
		InetSocketAddress serverAddress, Configuration clientConfig, JobID jobId, List<Path> files)
		throws IOException {

	checkNotNull(jobId);

	if (files.isEmpty()) {
		return Collections.emptyList();
	} else {
		List<PermanentBlobKey> blobKeys = new ArrayList<>();

		try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) {
			for (final Path file : files) {
				final PermanentBlobKey key = blobClient.uploadFile(jobId, file);
				blobKeys.add(key);
			}
		}

		return blobKeys;
	}
}
 
Example 15
Source Project: flink   Source File: ContinuousFileProcessingTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		File hdfsDir = tempFolder.newFolder();

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 
Example 16
Source Project: flink   Source File: FileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 *
 * @return The path of the file to read.
 *
 * @deprecated Please use getFilePaths() instead.
 */
@Deprecated
public Path getFilePath() {

	if (supportsMultiPaths()) {
		if (this.filePaths == null || this.filePaths.length == 0) {
			return null;
		} else if (this.filePaths.length == 1) {
			return this.filePaths[0];
		} else {
			throw new UnsupportedOperationException(
				"FileInputFormat is configured with multiple paths. Use getFilePaths() instead.");
		}
	} else {
		return filePath;
	}
}
 
Example 17
Source Project: flink   Source File: BootstrapTransformationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorSpecificMaxParallelismRespected() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	DataSource<Integer> input = env.fromElements(0);

	BootstrapTransformation<Integer> transformation = OperatorTransformation
		.bootstrapWith(input)
		.setMaxParallelism(1)
		.transform(new ExampleStateBootstrapFunction());

	int maxParallelism = transformation.getMaxParallelism(4);
	DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(
		OperatorIDGenerator.fromUid("uid"),
		new MemoryStateBackend(),
		new Path(),
		maxParallelism
	);

	Assert.assertEquals("The parallelism of a data set should be constrained my the savepoint max parallelism", 1, getParallelism(result));
}
 
Example 18
private IncrementalLocalKeyedStateHandle transferRemoteStateToLocalDirectory(
	Path temporaryRestoreInstancePath,
	IncrementalRemoteKeyedStateHandle restoreStateHandle) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	// since we transferred all remote state to a local directory, we can use the same code as for
	// local recovery.
	return new IncrementalLocalKeyedStateHandle(
		restoreStateHandle.getBackendIdentifier(),
		restoreStateHandle.getCheckpointId(),
		new DirectoryStateHandle(temporaryRestoreInstancePath),
		restoreStateHandle.getKeyGroupRange(),
		restoreStateHandle.getMetaStateHandle(),
		restoreStateHandle.getSharedState().keySet());
}
 
Example 19
Source Project: flink   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testParametrizationDirectories() throws Exception {
	final JobID jid = new JobID();
	final Path checkpointPath = new Path(tmp.newFolder().toURI().toString());
	final Path savepointPath = new Path(tmp.newFolder().toURI().toString());

	MemoryStateBackend backend = new MemoryStateBackend(
			checkpointPath.toString(), savepointPath.toString());

	MemoryBackendCheckpointStorage storage =
			(MemoryBackendCheckpointStorage) backend.createCheckpointStorage(jid);

	assertTrue(storage.supportsHighlyAvailableStorage());
	assertTrue(storage.hasDefaultSavepointLocation());
	assertNotNull(storage.getDefaultSavepointDirectory());

	assertEquals(savepointPath, storage.getDefaultSavepointDirectory());
}
 
Example 20
@Test
public void testSetFinishedAndRunning() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
 
Example 21
Source Project: flink   Source File: ContinuousFileProcessingTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		File hdfsDir = tempFolder.newFolder();

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: FsJobArchivist.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads the given archive file and returns a {@link Collection} of contained {@link ArchivedJson}.
 *
 * @param file archive to extract
 * @return collection of archived jsons
 * @throws IOException if the file can't be opened, read or doesn't contain valid json
 */
public static Collection<ArchivedJson> getArchivedJsons(Path file) throws IOException {
	try (FSDataInputStream input = file.getFileSystem().open(file);
		ByteArrayOutputStream output = new ByteArrayOutputStream()) {
		IOUtils.copyBytes(input, output);

		JsonNode archive = mapper.readTree(output.toByteArray());

		Collection<ArchivedJson> archives = new ArrayList<>();
		for (JsonNode archivePart : archive.get(ARCHIVE)) {
			String path = archivePart.get(PATH).asText();
			String json = archivePart.get(JSON).asText();
			archives.add(new ArchivedJson(path, json));
		}
		return archives;
	}
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ExecutionEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
public <X> DataSource<X> readFile(FileInputFormat<X> inputFormat, String filePath) {
	if (inputFormat == null) {
		throw new IllegalArgumentException("InputFormat must not be null.");
	}
	if (filePath == null) {
		throw new IllegalArgumentException("The file path must not be null.");
	}

	inputFormat.setFilePath(new Path(filePath));
	try {
		return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat));
	}
	catch (Exception e) {
		throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " +
				"Please specify the TypeInformation of the produced type explicitly by using the " +
				"'createInput(InputFormat, TypeInformation)' method instead.");
	}
}
 
Example 24
Source Project: flink   Source File: NotifyCheckpointAbortedITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName());

	checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI());
	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1).build());
	cluster.before();

	NormalMap.reset();
	DeclineSink.reset();
	TestingCompletedCheckpointStore.reset();
}
 
Example 25
Source Project: flink   Source File: AvroTypeExtractionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializeWithAvro() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			.map((MapFunction<User, User>) value -> {
				Map<CharSequence, Long> ab = new HashMap<>(1);
				ab.put("hehe", 12L);
				value.setTypeMap(ab);
				return value;
			});

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, " +
		"\"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, " +
		"\"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", " +
		"\"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, \"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, " +
		"\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, \"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, " +
		"\"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n" +
		"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, " +
		"\"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, " +
		"\"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, " +
		"\"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, \"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, " +
		"\"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, \"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, " +
		"\"type_decimal_fixed\": [7, -48]}\n";

}
 
Example 26
Source Project: Flink-CEPplus   Source File: CsvOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates an instance of CsvOutputFormat.
 *
 * @param outputPath The path where the CSV file is written.
 * @param recordDelimiter
 *            The delimiter that is used to separate the tuples.
 * @param fieldDelimiter
 *            The delimiter that is used to separate fields in a tuple.
 */
public CsvOutputFormat(Path outputPath, String recordDelimiter, String fieldDelimiter) {
	super(outputPath);
	if (recordDelimiter == null) {
		throw new IllegalArgumentException("RecordDelmiter shall not be null.");
	}

	if (fieldDelimiter == null) {
		throw new IllegalArgumentException("FieldDelimiter shall not be null.");
	}

	this.fieldDelimiter = fieldDelimiter;
	this.recordDelimiter = recordDelimiter;
	this.allowNullValues = false;
}
 
Example 27
Source Project: incubator-iotdb   Source File: TsFileInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
public TsFileInputFormat(
		@Nullable String path,
		QueryExpression expression,
		RowRecordParser<T> parser,
		@Nullable TSFileConfig config) {
	super(path != null ? new Path(path) : null);
	this.expression = expression;
	this.parser = parser;
	this.config = config;
}
 
Example 28
Source Project: flink   Source File: ParquetMapInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testReadMapFromNestedRecord() throws IOException {
	Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> nested = TestUtil.getNestedRecordTestData();
	Path path = TestUtil.createTempParquetFile(tempRoot.getRoot(), TestUtil.NESTED_SCHEMA, Collections.singletonList(nested.f1));
	MessageType nestedType = SCHEMA_CONVERTER.convert(TestUtil.NESTED_SCHEMA);

	ParquetMapInputFormat inputFormat = new ParquetMapInputFormat(path, nestedType);
	inputFormat.setRuntimeContext(TestUtil.getMockRuntimeContext());

	FileInputSplit[] splits = inputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	inputFormat.open(splits[0]);

	Map map = inputFormat.nextRecord(null);
	assertNotNull(map);
	assertEquals(5, map.size());
	assertArrayEquals((Long[]) nested.f2.getField(3), (Long[]) map.get("arr"));
	assertArrayEquals((String[]) nested.f2.getField(4), (String[]) map.get("strArray"));

	Map<String, String> mapItem = (Map<String, String>) ((Map) map.get("nestedMap")).get("mapItem");
	assertEquals(2, mapItem.size());
	assertEquals("map", mapItem.get("type"));
	assertEquals("hashMap", mapItem.get("value"));

	List<Map<String, String>> nestedArray = (List<Map<String, String>>) map.get("nestedArray");
	assertEquals(1, nestedArray.size());
	assertEquals("color", nestedArray.get(0).get("type"));
	assertEquals(1L, nestedArray.get(0).get("value"));
}
 
Example 29
Source Project: Flink-CEPplus   Source File: FileInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetPathsWithoutSettingFirst() {
	final DummyFileInputFormat format = new DummyFileInputFormat();
	
	Path[] paths = format.getFilePaths();
	Assert.assertNotNull("Paths should not be null.", paths);
	Assert.assertEquals("Paths size should be 0.", 0, paths.length);
}
 
Example 30
Source Project: Flink-CEPplus   Source File: AvroTypeExtractionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleAvroRead() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			.map((value) -> value);

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, " +
		"\"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], " +
		"\"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": {\"KEY 2\": 17554, \"KEY 1\": 8546456}, " +
		"\"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, " +
		"\"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, \"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, " +
		"\"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, " +
		"\"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n" +
		"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, " +
		"\"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, " +
		"\"type_enum\": \"RED\", \"type_map\": {}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", " +
		"\"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, \"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, " +
		"\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, \"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, " +
		"\"type_timestamp_micros\": 123456, \"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n";
}