Java Code Examples for org.apache.hadoop.hbase.HColumnDescriptor#setDataBlockEncoding()

The following examples show how to use org.apache.hadoop.hbase.HColumnDescriptor#setDataBlockEncoding() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HBaseBasedAuditRepository.java    From atlas with Apache License 2.0 6 votes vote down vote up 
private void createTableIfNotExists() throws AtlasException {
    Admin admin = null;
    try {
        admin = connection.getAdmin();
        LOG.info("Checking if table {} exists", tableName.getNameAsString());
        if (!admin.tableExists(tableName)) {
            LOG.info("Creating table {}", tableName.getNameAsString());
            HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
            HColumnDescriptor columnFamily = new HColumnDescriptor(COLUMN_FAMILY);
            columnFamily.setMaxVersions(1);
            columnFamily.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
            columnFamily.setCompressionType(Compression.Algorithm.GZ);
            columnFamily.setBloomFilterType(BloomType.ROW);
            tableDescriptor.addFamily(columnFamily);
            admin.createTable(tableDescriptor);
        } else {
            LOG.info("Table {} exists", tableName.getNameAsString());
        }
    } catch (IOException e) {
        throw new AtlasException(e);
    } finally {
        close(admin);
    }
}
Example 2
Source File: HBaseBasedAuditRepository.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up 
private void createTableIfNotExists() throws AtlasException {
    Admin admin = null;
    try {
        admin = connection.getAdmin();
        LOG.info("Checking if table {} exists", tableName.getNameAsString());
        if (!admin.tableExists(tableName)) {
            LOG.info("Creating table {}", tableName.getNameAsString());
            HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
            HColumnDescriptor columnFamily = new HColumnDescriptor(COLUMN_FAMILY);
            columnFamily.setMaxVersions(1);
            columnFamily.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
            columnFamily.setCompressionType(Compression.Algorithm.GZ);
            columnFamily.setBloomFilterType(BloomType.ROW);
            tableDescriptor.addFamily(columnFamily);
            admin.createTable(tableDescriptor);
        } else {
            LOG.info("Table {} exists", tableName.getNameAsString());
        }
    } catch (IOException e) {
        throw new AtlasException(e);
    } finally {
        close(admin);
    }
}
Example 3
Source File: ConnectionQueryServicesImpl.java    From phoenix with Apache License 2.0 6 votes vote down vote up 
private HColumnDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family.getFirst());
    if (tableType != PTableType.VIEW) {
        if(props.get(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB) != null){
            columnDesc.setKeepDeletedCells(props.getBoolean(
                QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_KEEP_DELETED_CELLS));
        }
        columnDesc.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING);
        for (Entry<String,Object> entry : family.getSecond().entrySet()) {
            String key = entry.getKey();
            Object value = entry.getValue();
            columnDesc.setValue(key, value == null ? null : value.toString());
        }
    }
    return columnDesc;
}
Example 4
Source File: HBaseStoreManager.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up 
private void setCFOptions(HColumnDescriptor cdesc, int ttlInSeconds) {
    if (null != compression && !compression.equals(COMPRESSION_DEFAULT))
        compat.setCompression(cdesc, compression);

    if (ttlInSeconds > 0)
        cdesc.setTimeToLive(ttlInSeconds);

    cdesc.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
}
Example 5
Source File: ConnectionQueryServicesImpl.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
private HColumnDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family.getFirst());
    if (tableType != PTableType.VIEW) {
        columnDesc.setKeepDeletedCells(true);
        columnDesc.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING);
        for (Entry<String,Object> entry : family.getSecond().entrySet()) {
            String key = entry.getKey();
            Object value = entry.getValue();
            columnDesc.setValue(key, value == null ? null : value.toString());
        }
    }
    return columnDesc;
}
Example 6
Source File: CubeHTableUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static HColumnDescriptor createColumnFamily(KylinConfig kylinConfig, String cfName, boolean isMemoryHungry) {
    HColumnDescriptor cf = new HColumnDescriptor(cfName);
    cf.setMaxVersions(1);

    if (isMemoryHungry) {
        cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize());
    } else {
        cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize());
    }

    String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase(Locale.ROOT);
    switch (hbaseDefaultCC) {
    case "snappy": {
        logger.info("hbase will use snappy to compress data");
        cf.setCompressionType(Algorithm.SNAPPY);
        break;
    }
    case "lzo": {
        logger.info("hbase will use lzo to compress data");
        cf.setCompressionType(Algorithm.LZO);
        break;
    }
    case "gz":
    case "gzip": {
        logger.info("hbase will use gzip to compress data");
        cf.setCompressionType(Algorithm.GZ);
        break;
    }
    case "lz4": {
        logger.info("hbase will use lz4 to compress data");
        cf.setCompressionType(Algorithm.LZ4);
        break;
    }
    case "none":
    default: {
        logger.info("hbase will not use any compression algorithm to compress data");
        cf.setCompressionType(Algorithm.NONE);
    }
    }

    try {
        String encodingStr = kylinConfig.getHbaseDefaultEncoding();
        DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
        cf.setDataBlockEncoding(encoding);
    } catch (Exception e) {
        logger.info("hbase will not use any encoding", e);
        cf.setDataBlockEncoding(DataBlockEncoding.NONE);
    }

    cf.setInMemory(false);
    cf.setBloomFilterType(BloomType.NONE);
    cf.setScope(kylinConfig.getHBaseReplicationScope());
    return cf;
}
Example 7
Source File: CubeHTableUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static HColumnDescriptor createColumnFamily(KylinConfig kylinConfig, String cfName, boolean isMemoryHungry) {
    HColumnDescriptor cf = new HColumnDescriptor(cfName);
    cf.setMaxVersions(1);

    if (isMemoryHungry) {
        cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize());
    } else {
        cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize());
    }

    String hbaseDefaultCC = kylinConfig.getHbaseDefaultCompressionCodec().toLowerCase(Locale.ROOT);
    switch (hbaseDefaultCC) {
    case "snappy": {
        logger.info("hbase will use snappy to compress data");
        cf.setCompressionType(Algorithm.SNAPPY);
        break;
    }
    case "lzo": {
        logger.info("hbase will use lzo to compress data");
        cf.setCompressionType(Algorithm.LZO);
        break;
    }
    case "gz":
    case "gzip": {
        logger.info("hbase will use gzip to compress data");
        cf.setCompressionType(Algorithm.GZ);
        break;
    }
    case "lz4": {
        logger.info("hbase will use lz4 to compress data");
        cf.setCompressionType(Algorithm.LZ4);
        break;
    }
    case "none":
    default: {
        logger.info("hbase will not use any compression algorithm to compress data");
        cf.setCompressionType(Algorithm.NONE);
    }
    }

    try {
        String encodingStr = kylinConfig.getHbaseDefaultEncoding();
        DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr);
        cf.setDataBlockEncoding(encoding);
    } catch (Exception e) {
        logger.info("hbase will not use any encoding", e);
        cf.setDataBlockEncoding(DataBlockEncoding.NONE);
    }

    cf.setInMemory(false);
    cf.setBloomFilterType(BloomType.NONE);
    cf.setScope(kylinConfig.getHBaseReplicationScope());
    return cf;
}
Example 8
Source File: CreateHTableJob.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OPTION_CUBE_NAME);
    options.addOption(OPTION_PARTITION_FILE_PATH);
    options.addOption(OPTION_HTABLE_NAME);
    parseOptions(options, args);

    Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));

    String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    CubeInstance cube = cubeMgr.getCube(cubeName);
    CubeDesc cubeDesc = cube.getDescriptor();

    String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
    HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
    // https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html
    tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());
    tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());

    Configuration conf = HBaseConfiguration.create(getConf());
    HBaseAdmin admin = new HBaseAdmin(conf);

    try {
        if (User.isHBaseSecurityEnabled(conf)) {
            // add coprocessor for bulk load
            tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
        }

        for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
            HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName());
            cf.setMaxVersions(1);

            if (LZOSupportnessChecker.getSupportness()) {
                logger.info("hbase will use lzo to compress data");
                cf.setCompressionType(Algorithm.LZO);
            } else {
                logger.info("hbase will not use lzo to compress data");
            }

            cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
            cf.setInMemory(false);
            cf.setBlocksize(4 * 1024 * 1024); // set to 4MB
            tableDesc.addFamily(cf);
        }

        byte[][] splitKeys = getSplits(conf, partitionFilePath);

        if (admin.tableExists(tableName)) {
            // admin.disableTable(tableName);
            // admin.deleteTable(tableName);
            throw new RuntimeException("HBase table " + tableName + " exists!");
        }

        DeployCoprocessorCLI.deployCoprocessor(tableDesc);

        admin.createTable(tableDesc, splitKeys);
        logger.info("create hbase table " + tableName + " done.");

        return 0;
    } catch (Exception e) {
        printUsage(options);
        e.printStackTrace(System.err);
        logger.error(e.getLocalizedMessage(), e);
        return 2;
    } finally {
        admin.close();
    }
}
Example 9
Source File: IICreateHTableJob.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    try {
        options.addOption(OPTION_II_NAME);
        options.addOption(OPTION_HTABLE_NAME);
        parseOptions(options, args);

        String tableName = getOptionValue(OPTION_HTABLE_NAME);
        String iiName = getOptionValue(OPTION_II_NAME);

        KylinConfig config = KylinConfig.getInstanceFromEnv();
        IIManager iiManager = IIManager.getInstance(config);
        IIInstance ii = iiManager.getII(iiName);
        int sharding = ii.getDescriptor().getSharding();

        HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
        HColumnDescriptor cf = new HColumnDescriptor(IIDesc.HBASE_FAMILY);
        cf.setMaxVersions(1);
        //cf.setCompressionType(Algorithm.LZO);
        cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
        tableDesc.addFamily(cf);
        tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());

        Configuration conf = HBaseConfiguration.create(getConf());
        if (User.isHBaseSecurityEnabled(conf)) {
            // add coprocessor for bulk load
            tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
        }

        DeployCoprocessorCLI.deployCoprocessor(tableDesc);

        // drop the table first
        HBaseAdmin admin = new HBaseAdmin(conf);
        if (admin.tableExists(tableName)) {
            admin.disableTable(tableName);
            admin.deleteTable(tableName);
        }

        // create table
        byte[][] splitKeys = getSplits(sharding);
        if (splitKeys.length == 0)
            splitKeys = null;
        admin.createTable(tableDesc, splitKeys);
        if (splitKeys != null) {
            for (int i = 0; i < splitKeys.length; i++) {
                System.out.println("split key " + i + ": " + BytesUtil.toHex(splitKeys[i]));
            }
        }
        System.out.println("create hbase table " + tableName + " done.");
        admin.close();

        return 0;
    } catch (Exception e) {
        printUsage(options);
        throw e;
    }
}
Example 10
Source File: TableCommand.java    From pinpoint with Apache License 2.0 4 votes vote down vote up 
private HColumnDescriptor newColumnDescriptor(ColumnFamilyChange columnFamilyChange) {
    HColumnDescriptor hcd = new HColumnDescriptor(columnFamilyChange.getName());
    ColumnFamilyConfiguration columnFamilyConfiguration = columnFamilyChange.getColumnFamilyConfiguration();
    Boolean blockCacheEnabled = columnFamilyConfiguration.getBlockCacheEnabled();
    if (blockCacheEnabled != null) {
        hcd.setBlockCacheEnabled(blockCacheEnabled);
    }
    Integer replicationScope = columnFamilyConfiguration.getReplicationScope();
    if (replicationScope != null) {
        hcd.setScope(replicationScope);
    }
    Boolean inMemory = columnFamilyConfiguration.getInMemory();
    if (inMemory != null) {
        hcd.setInMemory(inMemory);
    }
    Integer timeToLive = columnFamilyConfiguration.getTimeToLive();
    if (timeToLive != null) {
        hcd.setTimeToLive(timeToLive);
    }
    ColumnFamilyConfiguration.DataBlockEncoding dataBlockEncoding =
            columnFamilyConfiguration.getDataBlockEncoding();
    if (dataBlockEncoding != null) {
        hcd.setDataBlockEncoding(DataBlockEncoding.valueOf(dataBlockEncoding.name()));
    }
    Integer blockSize = columnFamilyConfiguration.getBlockSize();
    if (blockSize != null) {
        hcd.setBlocksize(blockSize);
    }
    Integer maxVersions = columnFamilyConfiguration.getMaxVersions();
    if (maxVersions != null) {
        hcd.setMaxVersions(maxVersions);
    }
    Integer minVersions = columnFamilyConfiguration.getMinVersions();
    if (minVersions != null) {
        hcd.setMinVersions(minVersions);
    }
    ColumnFamilyConfiguration.BloomFilter bloomFilter = columnFamilyConfiguration.getBloomFilter();
    if (bloomFilter != null) {
        hcd.setBloomFilterType(BloomType.valueOf(bloomFilter.name()));
    }
    if (compressionAlgorithm != Compression.Algorithm.NONE) {
        hcd.setCompressionType(compressionAlgorithm);
    }
    return hcd;
}