org.apache.flink.core.fs.Path Java Examples

@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 

/**
 * Creates an OrcRowInputFormat.
 *
 * @param path The path to read ORC files from.
 * @param orcSchema The schema of the ORC files as ORC TypeDescription.
 * @param orcConfig The configuration to read the ORC files with.
 * @param batchSize The number of Row objects to read in a batch.
 */
public OrcRowInputFormat(String path, TypeDescription orcSchema, Configuration orcConfig, int batchSize) {
	super(new Path(path));

	// configure OrcRowInputFormat
	this.schema = orcSchema;
	this.rowType = (RowTypeInfo) OrcBatchReader.schemaToTypeInfo(schema);
	this.conf = orcConfig;
	this.batchSize = batchSize;

	// set default selection mask, i.e., all fields.
	this.selectedFields = new int[this.schema.getChildren().size()];
	for (int i = 0; i < selectedFields.length; i++) {
		this.selectedFields[i] = i;
	}
}
 

public HadoopPathBasedBulkFormatBuilder(
	org.apache.hadoop.fs.Path basePath,
	HadoopPathBasedBulkWriter.Factory<IN> writerFactory,
	HadoopFileCommitterFactory fileCommitterFactory,
	Configuration configuration,
	BucketAssigner<IN, BucketID> assigner,
	CheckpointRollingPolicy<IN, BucketID> policy,
	BucketFactory<IN, BucketID> bucketFactory,
	OutputFileConfig outputFileConfig) {

	this.basePath = new Path(Preconditions.checkNotNull(basePath).toString());
	this.writerFactory = writerFactory;
	this.fileCommitterFactory = fileCommitterFactory;
	this.serializableConfiguration = new SerializableConfiguration(configuration);
	this.bucketAssigner = Preconditions.checkNotNull(assigner);
	this.rollingPolicy = Preconditions.checkNotNull(policy);
	this.bucketFactory = Preconditions.checkNotNull(bucketFactory);
	this.outputFileConfig = Preconditions.checkNotNull(outputFileConfig);
}
 

private void verifyPythonDriverOptionsParsing(final String[] args) throws FlinkParseException {
	final PythonDriverOptions pythonCommandOptions = commandLineParser.parse(args);

	// verify the parsed python entrypoint module
	assertEquals("xxx", pythonCommandOptions.getEntrypointModule());

	// verify the parsed python library files
	final List<Path> pythonMainFile = pythonCommandOptions.getPythonLibFiles();
	assertNotNull(pythonMainFile);
	assertEquals(4, pythonMainFile.size());
	assertEquals(
		pythonMainFile.stream().map(Path::getName).collect(Collectors.joining(",")),
		"xxx.py,a.py,b.py,c.py");

	// verify the python program arguments
	final List<String> programArgs = pythonCommandOptions.getProgramArgs();
	assertEquals(2, programArgs.size());
	assertEquals("--input", programArgs.get(0));
	assertEquals("in.txt", programArgs.get(1));
}
 

@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	Assume.assumeTrue("Swift container not configured, skipping test...", CONTAINER != null);
	Assume.assumeTrue("Swift username not configured, skipping test...", USERNAME != null);
	Assume.assumeTrue("Swift password not configured, skipping test...", PASSWORD != null);
	Assume.assumeTrue("Swift tenant not configured, skipping test...", TENANT != null);
	Assume.assumeTrue("Swift region not configured, skipping test...", REGION != null);

	// initialize configuration with valid credentials
	final Configuration conf = createConfiguration();

	FileSystem.initialize(conf);

	// check for uniqueness of the test directory
	final Path directory = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR);
	final FileSystem fs = directory.getFileSystem();

	// directory must not yet exist
	assertFalse(fs.exists(directory));

	// reset configuration
	FileSystem.initialize(new Configuration());

	skipTest = false;
}
 

@Override
public FileStatus[] listStatus(final Path f) throws IOException {

	final File localf = pathToFile(f);
	FileStatus[] results;

	if (!localf.exists()) {
		return null;
	}
	if (localf.isFile()) {
		return new FileStatus[] { new LocalFileStatus(localf, this) };
	}

	final String[] names = localf.list();
	if (names == null) {
		return null;
	}
	results = new FileStatus[names.length];
	for (int i = 0; i < names.length; i++) {
		results[i] = getFileStatus(new Path(f, names[i]));
	}

	return results;
}
 

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 

@Override
public JobExecutionResult execute(String jobName) throws Exception {
	OptimizedPlan op = compileProgram(jobName);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jobGraph = jgg.compileJobGraph(op);

	for (Path jarFile: jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph);
	return this.lastJobExecutionResult;
}
 

@Override
public Bucket<IN, BucketID> getNewBucket(
		final RecoverableWriter fsWriter,
		final int subtaskIndex,
		final BucketID bucketId,
		final Path bucketPath,
		final long initialPartCounter,
		final PartFileWriter.PartFileFactory<IN, BucketID> partFileWriterFactory,
		final RollingPolicy<IN, BucketID> rollingPolicy) {

	return Bucket.getNew(
			fsWriter,
			subtaskIndex,
			bucketId,
			bucketPath,
			initialPartCounter,
			partFileWriterFactory,
			rollingPolicy);
}
 

/** Create a file and fill it with content. */
private Tuple2<org.apache.hadoop.fs.Path, String> fillWithData(
	String base, String fileName, int fileIdx, String sampleLine) throws IOException, InterruptedException {

	assert (hdfs != null);

	org.apache.hadoop.fs.Path tmp =
		new org.apache.hadoop.fs.Path(base + "/." + fileName + fileIdx);

	FSDataOutputStream stream = hdfs.create(tmp);
	StringBuilder str = new StringBuilder();
	for (int i = 0; i < LINES_PER_FILE; i++) {
		String line = fileIdx + ": " + sampleLine + " " + i + "\n";
		str.append(line);
		stream.write(line.getBytes(ConfigConstants.DEFAULT_CHARSET));
	}
	stream.close();
	return new Tuple2<>(tmp, str.toString());
}
 

@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 

@Test
public void testReadPojoFromSimpleRecord() throws IOException {
	Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> simple = TestUtil.getSimpleRecordTestData();
	MessageType messageType = SCHEMA_CONVERTER.convert(TestUtil.SIMPLE_SCHEMA);
	Path path = TestUtil.createTempParquetFile(tempRoot.getRoot(), TestUtil.SIMPLE_SCHEMA, Collections.singletonList(simple.f1));

	ParquetPojoInputFormat<PojoSimpleRecord> inputFormat = new ParquetPojoInputFormat<>(
		path, messageType, (PojoTypeInfo<PojoSimpleRecord>) Types.POJO(PojoSimpleRecord.class));
	inputFormat.setRuntimeContext(TestUtil.getMockRuntimeContext());

	FileInputSplit[] splits = inputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	inputFormat.open(splits[0]);

	PojoSimpleRecord simpleRecord = inputFormat.nextRecord(null);
	assertEquals(simple.f2.getField(0), simpleRecord.getFoo());
	assertEquals(simple.f2.getField(1), simpleRecord.getBar());
	assertArrayEquals((Long[]) simple.f2.getField(2), simpleRecord.getArr());
}
 

/**
 * Uploads the JAR files to the {@link PermanentBlobService} of the {@link BlobServer} at the
 * given address with HA as configured.
 *
 * @param serverAddress
 * 		Server address of the {@link BlobServer}
 * @param clientConfig
 * 		Any additional configuration for the blob client
 * @param jobId
 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
 * @param files
 * 		List of files to upload
 *
 * @throws IOException
 * 		if the upload fails
 */
public static List<PermanentBlobKey> uploadFiles(
		InetSocketAddress serverAddress, Configuration clientConfig, JobID jobId, List<Path> files)
		throws IOException {

	checkNotNull(jobId);

	if (files.isEmpty()) {
		return Collections.emptyList();
	} else {
		List<PermanentBlobKey> blobKeys = new ArrayList<>();

		try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) {
			for (final Path file : files) {
				final PermanentBlobKey key = blobClient.uploadFile(jobId, file);
				blobKeys.add(key);
			}
		}

		return blobKeys;
	}
}
 

@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		File hdfsDir = tempFolder.newFolder();

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 

/**
 *
 * @return The path of the file to read.
 *
 * @deprecated Please use getFilePaths() instead.
 */
@Deprecated
public Path getFilePath() {

	if (supportsMultiPaths()) {
		if (this.filePaths == null || this.filePaths.length == 0) {
			return null;
		} else if (this.filePaths.length == 1) {
			return this.filePaths[0];
		} else {
			throw new UnsupportedOperationException(
				"FileInputFormat is configured with multiple paths. Use getFilePaths() instead.");
		}
	} else {
		return filePath;
	}
}
 

@Test
public void testOperatorSpecificMaxParallelismRespected() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	DataSource<Integer> input = env.fromElements(0);

	BootstrapTransformation<Integer> transformation = OperatorTransformation
		.bootstrapWith(input)
		.setMaxParallelism(1)
		.transform(new ExampleStateBootstrapFunction());

	int maxParallelism = transformation.getMaxParallelism(4);
	DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(
		OperatorIDGenerator.fromUid("uid"),
		new MemoryStateBackend(),
		new Path(),
		maxParallelism
	);

	Assert.assertEquals("The parallelism of a data set should be constrained my the savepoint max parallelism", 1, getParallelism(result));
}
 

private IncrementalLocalKeyedStateHandle transferRemoteStateToLocalDirectory(
	Path temporaryRestoreInstancePath,
	IncrementalRemoteKeyedStateHandle restoreStateHandle) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	// since we transferred all remote state to a local directory, we can use the same code as for
	// local recovery.
	return new IncrementalLocalKeyedStateHandle(
		restoreStateHandle.getBackendIdentifier(),
		restoreStateHandle.getCheckpointId(),
		new DirectoryStateHandle(temporaryRestoreInstancePath),
		restoreStateHandle.getKeyGroupRange(),
		restoreStateHandle.getMetaStateHandle(),
		restoreStateHandle.getSharedState().keySet());
}
 

@Test
public void testParametrizationDirectories() throws Exception {
	final JobID jid = new JobID();
	final Path checkpointPath = new Path(tmp.newFolder().toURI().toString());
	final Path savepointPath = new Path(tmp.newFolder().toURI().toString());

	MemoryStateBackend backend = new MemoryStateBackend(
			checkpointPath.toString(), savepointPath.toString());

	MemoryBackendCheckpointStorage storage =
			(MemoryBackendCheckpointStorage) backend.createCheckpointStorage(jid);

	assertTrue(storage.supportsHighlyAvailableStorage());
	assertTrue(storage.hasDefaultSavepointLocation());
	assertNotNull(storage.getDefaultSavepointDirectory());

	assertEquals(savepointPath, storage.getDefaultSavepointDirectory());
}
 

@Test
public void testSetFinishedAndRunning() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
 

@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		File hdfsDir = tempFolder.newFolder();

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 

/**
 * Reads the given archive file and returns a {@link Collection} of contained {@link ArchivedJson}.
 *
 * @param file archive to extract
 * @return collection of archived jsons
 * @throws IOException if the file can't be opened, read or doesn't contain valid json
 */
public static Collection<ArchivedJson> getArchivedJsons(Path file) throws IOException {
	try (FSDataInputStream input = file.getFileSystem().open(file);
		ByteArrayOutputStream output = new ByteArrayOutputStream()) {
		IOUtils.copyBytes(input, output);

		JsonNode archive = mapper.readTree(output.toByteArray());

		Collection<ArchivedJson> archives = new ArrayList<>();
		for (JsonNode archivePart : archive.get(ARCHIVE)) {
			String path = archivePart.get(PATH).asText();
			String json = archivePart.get(JSON).asText();
			archives.add(new ArchivedJson(path, json));
		}
		return archives;
	}
}
 

public <X> DataSource<X> readFile(FileInputFormat<X> inputFormat, String filePath) {
	if (inputFormat == null) {
		throw new IllegalArgumentException("InputFormat must not be null.");
	}
	if (filePath == null) {
		throw new IllegalArgumentException("The file path must not be null.");
	}

	inputFormat.setFilePath(new Path(filePath));
	try {
		return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat));
	}
	catch (Exception e) {
		throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " +
				"Please specify the TypeInformation of the produced type explicitly by using the " +
				"'createInput(InputFormat, TypeInformation)' method instead.");
	}
}
 

@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName());

	checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI());
	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1).build());
	cluster.before();

	NormalMap.reset();
	DeclineSink.reset();
	TestingCompletedCheckpointStore.reset();
}
 

@Test
public void testSerializeWithAvro() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			.map((MapFunction<User, User>) value -> {
				Map<CharSequence, Long> ab = new HashMap<>(1);
				ab.put("hehe", 12L);
				value.setTypeMap(ab);
				return value;
			});

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, " +
		"\"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, " +
		"\"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", " +
		"\"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, \"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, " +
		"\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, \"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, " +
		"\"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n" +
		"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, " +
		"\"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, " +
		"\"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, " +
		"\"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, \"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, " +
		"\"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, \"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, " +
		"\"type_decimal_fixed\": [7, -48]}\n";

}
 

/**
 * Creates an instance of CsvOutputFormat.
 *
 * @param outputPath The path where the CSV file is written.
 * @param recordDelimiter
 *            The delimiter that is used to separate the tuples.
 * @param fieldDelimiter
 *            The delimiter that is used to separate fields in a tuple.
 */
public CsvOutputFormat(Path outputPath, String recordDelimiter, String fieldDelimiter) {
	super(outputPath);
	if (recordDelimiter == null) {
		throw new IllegalArgumentException("RecordDelmiter shall not be null.");
	}

	if (fieldDelimiter == null) {
		throw new IllegalArgumentException("FieldDelimiter shall not be null.");
	}

	this.fieldDelimiter = fieldDelimiter;
	this.recordDelimiter = recordDelimiter;
	this.allowNullValues = false;
}
 

public TsFileInputFormat(
		@Nullable String path,
		QueryExpression expression,
		RowRecordParser<T> parser,
		@Nullable TSFileConfig config) {
	super(path != null ? new Path(path) : null);
	this.expression = expression;
	this.parser = parser;
	this.config = config;
}
 

@Test
@SuppressWarnings("unchecked")
public void testReadMapFromNestedRecord() throws IOException {
	Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> nested = TestUtil.getNestedRecordTestData();
	Path path = TestUtil.createTempParquetFile(tempRoot.getRoot(), TestUtil.NESTED_SCHEMA, Collections.singletonList(nested.f1));
	MessageType nestedType = SCHEMA_CONVERTER.convert(TestUtil.NESTED_SCHEMA);

	ParquetMapInputFormat inputFormat = new ParquetMapInputFormat(path, nestedType);
	inputFormat.setRuntimeContext(TestUtil.getMockRuntimeContext());

	FileInputSplit[] splits = inputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	inputFormat.open(splits[0]);

	Map map = inputFormat.nextRecord(null);
	assertNotNull(map);
	assertEquals(5, map.size());
	assertArrayEquals((Long[]) nested.f2.getField(3), (Long[]) map.get("arr"));
	assertArrayEquals((String[]) nested.f2.getField(4), (String[]) map.get("strArray"));

	Map<String, String> mapItem = (Map<String, String>) ((Map) map.get("nestedMap")).get("mapItem");
	assertEquals(2, mapItem.size());
	assertEquals("map", mapItem.get("type"));
	assertEquals("hashMap", mapItem.get("value"));

	List<Map<String, String>> nestedArray = (List<Map<String, String>>) map.get("nestedArray");
	assertEquals(1, nestedArray.size());
	assertEquals("color", nestedArray.get(0).get("type"));
	assertEquals(1L, nestedArray.get(0).get("value"));
}
 

@Test
public void testGetPathsWithoutSettingFirst() {
	final DummyFileInputFormat format = new DummyFileInputFormat();
	
	Path[] paths = format.getFilePaths();
	Assert.assertNotNull("Paths should not be null.", paths);
	Assert.assertEquals("Paths size should be 0.", 0, paths.length);
}
 

@Test
public void testSimpleAvroRead() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			.map((value) -> value);

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, " +
		"\"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], " +
		"\"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": {\"KEY 2\": 17554, \"KEY 1\": 8546456}, " +
		"\"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, " +
		"\"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, \"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, " +
		"\"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, " +
		"\"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n" +
		"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, " +
		"\"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, " +
		"\"type_enum\": \"RED\", \"type_map\": {}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", " +
		"\"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, \"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, " +
		"\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, \"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, " +
		"\"type_timestamp_micros\": 123456, \"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n";
}