Java Code Examples for org.apache.parquet.hadoop.metadata.CompressionCodecName#fromCompressionCodec()

The following examples show how to use org.apache.parquet.hadoop.metadata.CompressionCodecName#fromCompressionCodec() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ParquetAsTextOutputFormat.java    From iow-hadoop-streaming with Apache License 2.0 6 votes vote down vote up 
private static CompressionCodecName getCodec(JobConf conf) {

        CompressionCodecName codec;

        if (ParquetOutputFormat.isCompressionSet(conf)) { // explicit parquet config
            codec = ParquetOutputFormat.getCompression(conf);
        } else if (getCompressOutput(conf)) { // from hadoop config
            // find the right codec
            Class<?> codecClass = getOutputCompressorClass(conf, DefaultCodec.class);
            LOG.info("Compression set through hadoop codec: " + codecClass.getName());
            codec = CompressionCodecName.fromCompressionCodec(codecClass);
        } else {
            codec = CompressionCodecName.UNCOMPRESSED;
        }

        LOG.info("Compression: " + codec.name());
        return codec;
    }
Example 2
Source File: CodecFactory.java    From pxf with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the {@link CompressionCodecName} for the given name, or default if name is null
 *
 * @param name         the name or class name of the compression codec
 * @param defaultCodec the default codec
 * @return the {@link CompressionCodecName} for the given name, or default if name is null
 */
public CompressionCodecName getCodec(String name, CompressionCodecName defaultCodec) {
    if (name == null) return defaultCodec;

    try {
        return CompressionCodecName.fromConf(name);
    } catch (IllegalArgumentException ie) {
        try {
            return CompressionCodecName.fromCompressionCodec(Class.forName(name));
        } catch (ClassNotFoundException ce) {
            throw new IllegalArgumentException(String.format("Invalid codec: %s ", name));
        }
    }
}
Example 3
Source File: AvroParquetFileReaderWriterFactory.java    From secor with Apache License 2.0 5 votes vote down vote up 
public AvroParquetFileWriter(LogFilePath logFilePath, CompressionCodec codec) throws IOException {
    Path path = new Path(logFilePath.getLogFilePath());
    LOG.debug("Creating Brand new Writer for path {}", path);
    CompressionCodecName codecName = CompressionCodecName
            .fromCompressionCodec(codec != null ? codec.getClass() : null);
    topic = logFilePath.getTopic();
    // Not setting blockSize, pageSize, enableDictionary, and validating
    writer = AvroParquetWriter.builder(path)
            .withSchema(schemaRegistry.getSchema(topic))
            .withCompressionCodec(codecName)
            .build();
}
Example 4
Source File: ThriftParquetFileReaderWriterFactory.java    From secor with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings({ "rawtypes", "unchecked" })
public ThriftParquetFileWriter(LogFilePath logFilePath, CompressionCodec codec) throws IOException {
    Path path = new Path(logFilePath.getLogFilePath());
    CompressionCodecName codecName = CompressionCodecName.fromCompressionCodec(codec != null ? codec.getClass() : null);
    topic = logFilePath.getTopic();
    writer = new ThriftParquetWriter(path, thriftUtil.getMessageClass(topic), codecName,
            blockSize, pageSize, enableDictionary, validating);
}
Example 5
Source File: ProtobufParquetFileReaderWriterFactory.java    From secor with Apache License 2.0 5 votes vote down vote up 
public ProtobufParquetFileWriter(LogFilePath logFilePath, CompressionCodec codec) throws IOException {
    Path path = new Path(logFilePath.getLogFilePath());
    CompressionCodecName codecName = CompressionCodecName
            .fromCompressionCodec(codec != null ? codec.getClass() : null);
    topic = logFilePath.getTopic();
    writer = new ProtoParquetWriter<Message>(path, protobufUtil.getMessageClass(topic), codecName,
            blockSize, pageSize, enableDictionary, validating);
}
Example 6
Source File: AvroFileReaderWriterFactory.java    From secor with Apache License 2.0 5 votes vote down vote up 
private CodecFactory getCodecFactory(CompressionCodec codec) {
    CompressionCodecName codecName = CompressionCodecName
            .fromCompressionCodec(codec != null ? codec.getClass() : null);
    try {
        return CodecFactory.fromString(codecName.name().toLowerCase());
    } catch (AvroRuntimeException e) {
        LOG.error("Error creating codec factory", e);
    }
    return CodecFactory.fromString("null");
}