Java Code Examples for org.apache.parquet.hadoop.ParquetFileWriter#start()
The following examples show how to use
org.apache.parquet.hadoop.ParquetFileWriter#start() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TransCompressionCommand.java From parquet-mr with Apache License 2.0 | 7 votes |
@Override @SuppressWarnings("unchecked") public int run() throws IOException { Preconditions.checkArgument(input != null && output != null, "Both input and output parquet file paths are required."); Preconditions.checkArgument(codec != null, "The codec cannot be null"); Path inPath = new Path(input); Path outPath = new Path(output); CompressionCodecName codecName = CompressionCodecName.valueOf(codec); ParquetMetadata metaData = ParquetFileReader.readFooter(getConf(), inPath, NO_FILTER); MessageType schema = metaData.getFileMetaData().getSchema(); ParquetFileWriter writer = new ParquetFileWriter(getConf(), schema, outPath, ParquetFileWriter.Mode.CREATE); writer.start(); try (TransParquetFileReader reader = new TransParquetFileReader(HadoopInputFile.fromPath(inPath, getConf()), HadoopReadOptions.builder(getConf()).build())) { compressionConverter.processBlocks(reader, writer, metaData, schema, metaData.getFileMetaData().getCreatedBy(), codecName); } finally { writer.end(metaData.getFileMetaData().getKeyValueMetaData()); } return 0; }
Example 2
Source File: ParquetHdfsFileSink.java From components with Apache License 2.0 | 6 votes |
@Override protected void mergeOutput(FileSystem fs, String sourceFolder, String targetFile) throws IOException { FileStatus[] sourceStatuses = FileSystemUtil.listSubFiles(fs, sourceFolder); List<Path> sourceFiles = new ArrayList<>(); for (FileStatus sourceStatus : sourceStatuses) { sourceFiles.add(sourceStatus.getPath()); } FileMetaData mergedMeta = ParquetFileWriter.mergeMetadataFiles(sourceFiles, fs.getConf()).getFileMetaData(); ParquetFileWriter writer = new ParquetFileWriter(fs.getConf(), mergedMeta.getSchema(), new Path(targetFile), ParquetFileWriter.Mode.CREATE); writer.start(); for (Path input : sourceFiles) { writer.appendFile(fs.getConf(), input); } writer.end(mergedMeta.getKeyValueMetaData()); }
Example 3
Source File: TransCompressionCommand.java From parquet-mr with Apache License 2.0 | 6 votes |
@Override public void execute(CommandLine options) throws Exception { super.execute(options); List<String> args = options.getArgList(); Path inPath = new Path(args.get(0)); Path outPath = new Path(args.get(1)); CompressionCodecName codecName = CompressionCodecName.valueOf(args.get(2)); ParquetMetadata metaData = ParquetFileReader.readFooter(conf, inPath, NO_FILTER); MessageType schema = metaData.getFileMetaData().getSchema(); ParquetFileWriter writer = new ParquetFileWriter(conf, schema, outPath, ParquetFileWriter.Mode.CREATE); writer.start(); try (TransParquetFileReader reader = new TransParquetFileReader(HadoopInputFile.fromPath(inPath, conf), HadoopReadOptions.builder(conf).build())) { compressionConverter.processBlocks(reader, writer, metaData, schema, metaData.getFileMetaData().getCreatedBy(), codecName); } finally { writer.end(metaData.getFileMetaData().getKeyValueMetaData()); } }
Example 4
Source File: CompressionConveterTest.java From parquet-mr with Apache License 2.0 | 6 votes |
private void convertCompression(Configuration conf, String inputFile, String outputFile, String codec) throws IOException { Path inPath = new Path(inputFile); Path outPath = new Path(outputFile); CompressionCodecName codecName = CompressionCodecName.valueOf(codec); ParquetMetadata metaData = ParquetFileReader.readFooter(conf, inPath, NO_FILTER); MessageType schema = metaData.getFileMetaData().getSchema(); ParquetFileWriter writer = new ParquetFileWriter(conf, schema, outPath, ParquetFileWriter.Mode.CREATE); writer.start(); try (TransParquetFileReader reader = new TransParquetFileReader(HadoopInputFile.fromPath(inPath, conf), HadoopReadOptions.builder(conf).build())) { compressionConverter.processBlocks(reader, writer, metaData, schema, metaData.getFileMetaData().getCreatedBy(), codecName); } finally { writer.end(metaData.getFileMetaData().getKeyValueMetaData()); } }
Example 5
Source File: ProtoParquetWriterWithOffset.java From garmadon with Apache License 2.0 | 5 votes |
protected void mergeToFinalPath(Path lastAvailableFinalPath, Path finalPath) throws IOException { try (ParquetFileReader reader = ParquetFileReader.open(fs.getConf(), lastAvailableFinalPath)) { MessageType schema = reader.getFileMetaData().getSchema(); if (!checkSchemaEquality(schema)) { LOGGER.warn("Schema between last available final file ({}) and temp file ({}) are not identical. We can't merge them", lastAvailableFinalPath, temporaryHdfsPath); moveToFinalPath(temporaryHdfsPath, finalPath); } else { Path mergedTempFile = new Path(temporaryHdfsPath.toString() + ".merged"); if (fs.isFile(mergedTempFile)) fs.delete(mergedTempFile, false); Map<String, String> existingMetadata = reader.getFileMetaData().getKeyValueMetaData(); Map<String, String> newMetadata = new HashMap<>(existingMetadata); newMetadata.put(LATEST_TIMESTAMP_META_KEY, String.valueOf(latestTimestamp)); ParquetFileWriter writerPF = new ParquetFileWriter(fs.getConf(), schema, mergedTempFile); writerPF.start(); try ( ParquetFileReader dest = ParquetFileReader.open(fs.getConf(), lastAvailableFinalPath); ParquetFileReader temp = ParquetFileReader.open(fs.getConf(), temporaryHdfsPath) ) { dest.appendTo(writerPF); temp.appendTo(writerPF); writerPF.end(newMetadata); } moveToFinalPath(mergedTempFile, lastAvailableFinalPath); try { fs.delete(temporaryHdfsPath, false); // This file is in a temp folder that should be deleted at exit so we should not throw exception here } catch (IOException ignored) { } } } }
Example 6
Source File: ParquetRecordWriter.java From dremio-oss with Apache License 2.0 | 5 votes |
/** * Helper method to create a new {@link ParquetFileWriter} as impersonated user. * @throws IOException */ private void initRecordWriter() throws IOException { this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension)); parquetFileWriter = new ParquetFileWriter(OutputFile.of(fs, path), checkNotNull(schema), ParquetFileWriter.Mode.CREATE, DEFAULT_BLOCK_SIZE, MAX_PADDING_SIZE_DEFAULT, DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH, true); parquetFileWriter.start(); }
Example 7
Source File: PruneColumnsCommand.java From parquet-mr with Apache License 2.0 | 5 votes |
@Override public void execute(CommandLine options) throws Exception { List<String> args = options.getArgList(); Path inputFile = new Path(args.get(0)); Path outputFile = new Path(args.get(1)); List<String> cols = args.subList(2, args.size()); Set<ColumnPath> prunePaths = convertToColumnPaths(cols); ParquetMetadata pmd = ParquetFileReader.readFooter(conf, inputFile, ParquetMetadataConverter.NO_FILTER); FileMetaData metaData = pmd.getFileMetaData(); MessageType schema = metaData.getSchema(); List<String> paths = new ArrayList<>(); getPaths(schema, paths, null); for (String col : cols) { if (!paths.contains(col)) { LOG.warn("Input column name {} doesn't show up in the schema of file {}", col, inputFile.getName()); } } ParquetFileWriter writer = new ParquetFileWriter(conf, pruneColumnsInSchema(schema, prunePaths), outputFile, ParquetFileWriter.Mode.CREATE); writer.start(); writer.appendFile(HadoopInputFile.fromPath(inputFile, conf)); writer.end(metaData.getKeyValueMetaData()); }
Example 8
Source File: MergeCommand.java From parquet-mr with Apache License 2.0 | 5 votes |
@Override public void execute(CommandLine options) throws Exception { // Prepare arguments List<String> args = options.getArgList(); List<Path> inputFiles = getInputFiles(args.subList(0, args.size() - 1)); Path outputFile = new Path(args.get(args.size() - 1)); // Merge schema and extraMeta FileMetaData mergedMeta = mergedMetadata(inputFiles); PrintWriter out = new PrintWriter(Main.out, true); // Merge data ParquetFileWriter writer = new ParquetFileWriter(conf, mergedMeta.getSchema(), outputFile, ParquetFileWriter.Mode.CREATE); writer.start(); boolean tooSmallFilesMerged = false; for (Path input: inputFiles) { if (input.getFileSystem(conf).getFileStatus(input).getLen() < TOO_SMALL_FILE_THRESHOLD) { out.format("Warning: file %s is too small, length: %d\n", input, input.getFileSystem(conf).getFileStatus(input).getLen()); tooSmallFilesMerged = true; } writer.appendFile(HadoopInputFile.fromPath(input, conf)); } if (tooSmallFilesMerged) { out.println("Warning: you merged too small files. " + "Although the size of the merged file is bigger, it STILL contains small row groups, thus you don't have the advantage of big row groups, " + "which usually leads to bad query performance!"); } writer.end(mergedMeta.getKeyValueMetaData()); }