org.apache.hadoop.hbase.io.encoding.DataBlockEncoding Java Examples

The following examples show how to use org.apache.hadoop.hbase.io.encoding.DataBlockEncoding. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestHFileDataBlockEncoder.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
  List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
  HFileBlock block = getSampleHFileBlock(kvs, useTag);
  HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag);

  LruBlockCache blockCache =
      new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
  BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
  blockCache.cacheBlock(cacheKey, cacheBlock);

  HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
  assertTrue(heapSize instanceof HFileBlock);

  HFileBlock returnedBlock = (HFileBlock) heapSize;

  if (blockEncoder.getDataBlockEncoding() ==
      DataBlockEncoding.NONE) {
    assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly());
  } else {
    if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
      System.out.println(blockEncoder);
    }
    assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
  }
}
 
Example #2
Source File: TestMajorCompaction.java    From hbase with Apache License 2.0 6 votes vote down vote up
public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception {
  Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>();
  for (HStore store : r.getStores()) {
    HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
    replaceBlockCache.put(store, blockEncoder);
    final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
    final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache;
    ((HStore) store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk));
  }

  majorCompaction();

  // restore settings
  for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) {
    ((HStore) entry.getKey()).setDataBlockEncoderInTest(entry.getValue());
  }
}
 
Example #3
Source File: TestHFileScannerImplReferenceCount.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void writeHFile(Configuration conf, FileSystem fs, Path hfilePath, Algorithm compression,
    DataBlockEncoding encoding, int cellCount) throws IOException {
  HFileContext context =
      new HFileContextBuilder().withBlockSize(1).withDataBlockEncoding(DataBlockEncoding.NONE)
          .withCompression(compression).withDataBlockEncoding(encoding).build();
  try (HFile.Writer writer =
      new HFile.WriterFactory(conf, new CacheConfig(conf)).withPath(fs, hfilePath)
          .withFileContext(context).create()) {
    Random rand = new Random(9713312); // Just a fixed seed.
    for (int i = 0; i < cellCount; ++i) {
      byte[] keyBytes = Bytes.add(Bytes.toBytes(i), SUFFIX);

      // A random-length random value.
      byte[] valueBytes = RandomKeyValueUtil.randomValue(rand);
      KeyValue keyValue =
          new KeyValue(keyBytes, FAMILY, QUALIFIER, HConstants.LATEST_TIMESTAMP, valueBytes);
      if (firstCell == null) {
        firstCell = keyValue;
      } else if (secondCell == null) {
        secondCell = keyValue;
      }
      writer.append(keyValue);
    }
  }
}
 
Example #4
Source File: TestHStoreFile.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Check if data block encoding information is saved correctly in HFile's file info.
 */
@Test
public void testDataBlockEncodingMetaData() throws IOException {
  // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
  Path dir = new Path(new Path(testDir, "7e0102"), "familyname");
  Path path = new Path(dir, "1234567890");

  DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF;
  cacheConf = new CacheConfig(conf);
  HFileContext meta =
    new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE)
      .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();
  writer.close();

  HStoreFile storeFile =
    new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);
  storeFile.initReader();
  StoreFileReader reader = storeFile.getReader();

  Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
  byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
  assertArrayEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
}
 
Example #5
Source File: TestHFileScannerImplReferenceCount.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testDisabledBlockCache() throws Exception {
  writeHFile(conf, fs, hfilePath, Algorithm.NONE, DataBlockEncoding.NONE, CELL_COUNT);
  // Set LruBlockCache
  conf.setFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
  BlockCache defaultBC = BlockCacheFactory.createBlockCache(conf);
  Assert.assertNull(defaultBC);
  CacheConfig cacheConfig = new CacheConfig(conf, null, defaultBC, allocator);
  Assert.assertFalse(cacheConfig.isCombinedBlockCache()); // Must be LruBlockCache.
  HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConfig, true, conf);
  Assert.assertTrue(reader instanceof HFileReaderImpl);
  // We've build a HFile tree with index = 16.
  Assert.assertEquals(16, reader.getTrailer().getNumDataIndexLevels());

  HFileBlock block1 = reader.getDataBlockIndexReader()
      .loadDataBlockWithScanInfo(firstCell, null, true, true, false,
          DataBlockEncoding.NONE, reader).getHFileBlock();

  Assert.assertTrue(block1.isSharedMem());
  Assert.assertTrue(block1 instanceof SharedMemHFileBlock);
  Assert.assertEquals(1, block1.refCnt());
  Assert.assertTrue(block1.release());
}
 
Example #6
Source File: HFileContext.java    From hbase with Apache License 2.0 6 votes vote down vote up
HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
             Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
             int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
             Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
             byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
  this.usesHBaseChecksum = useHBaseChecksum;
  this.includesMvcc =  includesMvcc;
  this.includesTags = includesTags;
  this.compressAlgo = compressAlgo;
  this.compressTags = compressTags;
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
  this.blocksize = blockSize;
  if (encoding != null) {
    this.encoding = encoding;
  }
  this.cryptoContext = cryptoContext;
  this.fileCreateTime = fileCreateTime;
  this.hfileName = hfileName;
  this.columnFamily = columnFamily;
  this.tableName = tableName;
  // If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
  // be the meta table comparator. Comparators are per table.
  this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
    CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
}
 
Example #7
Source File: HFileReaderImpl.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Updates the current block to be the given {@link HFileBlock}. Seeks to the the first
 * key/value pair.
 * @param newBlock the block to make current, and read by {@link HFileReaderImpl#readBlock},
 *          it's a totally new block with new allocated {@link ByteBuff}, so if no further
 *          reference to this block, we should release it carefully.
 */
@Override
protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
  try {
    // sanity checks
    if (newBlock.getBlockType() != BlockType.ENCODED_DATA) {
      throw new IllegalStateException("EncodedScanner works only on encoded data blocks");
    }
    short dataBlockEncoderId = newBlock.getDataBlockEncodingId();
    if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
      String encoderCls = dataBlockEncoder.getClass().getName();
      throw new CorruptHFileException("Encoder " + encoderCls +
        " doesn't support data block encoding " +
        DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
    }
    updateCurrBlockRef(newBlock);
    ByteBuff encodedBuffer = getEncodedBuffer(newBlock);
    seeker.setCurrentBuffer(encodedBuffer);
  } finally {
    releaseIfNotCurBlock(newBlock);
  }
  // Reset the next indexed key
  this.nextIndexedKey = null;
}
 
Example #8
Source File: HFileDataBlockEncoderImpl.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static HFileDataBlockEncoder createFromFileInfo(
    HFileInfo fileInfo) throws IOException {
  DataBlockEncoding encoding = DataBlockEncoding.NONE;
  byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
  if (dataBlockEncodingType != null) {
    String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
    try {
      encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
    } catch (IllegalArgumentException ex) {
      throw new IOException("Invalid data block encoding type in file info: "
        + dataBlockEncodingStr, ex);
    }
  }

  if (encoding == DataBlockEncoding.NONE) {
    return NoOpDataBlockEncoder.INSTANCE;
  }
  return new HFileDataBlockEncoderImpl(encoding);
}
 
Example #9
Source File: TestMobDataBlockEncoding.java    From hbase with Apache License 2.0 6 votes vote down vote up
public void setUp(long threshold, String TN, DataBlockEncoding encoding)
    throws Exception {
  tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(TN));
  columnFamilyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
  columnFamilyDescriptor.setMobEnabled(true);
  columnFamilyDescriptor.setMobThreshold(threshold);
  columnFamilyDescriptor.setMaxVersions(4);
  columnFamilyDescriptor.setDataBlockEncoding(encoding);
  tableDescriptor.setColumnFamily(columnFamilyDescriptor);
  admin = TEST_UTIL.getAdmin();
  admin.createTable(tableDescriptor);
  table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
    .getTable(TableName.valueOf(TN));
}
 
Example #10
Source File: HFileWriterImpl.java    From hbase with Apache License 2.0 6 votes vote down vote up
public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path,
    FSDataOutputStream outputStream, HFileContext fileContext) {
  this.outputStream = outputStream;
  this.path = path;
  this.name = path != null ? path.getName() : outputStream.toString();
  this.hFileContext = fileContext;
  DataBlockEncoding encoding = hFileContext.getDataBlockEncoding();
  if (encoding != DataBlockEncoding.NONE) {
    this.blockEncoder = new HFileDataBlockEncoderImpl(encoding);
  } else {
    this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
  }
  closeOutputStream = path != null;
  this.cacheConf = cacheConf;
  float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f);
  this.encodedBlockSizeLimit = (int)(hFileContext.getBlocksize() * encodeBlockSizeRatio);
  finishInit(conf);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Writer" + (path != null ? " for " + path : "") +
      " initialized with cacheConf: " + cacheConf +
      " fileContext: " + fileContext);
  }
}
 
Example #11
Source File: TestMobDataBlockEncoding.java    From hbase with Apache License 2.0 6 votes vote down vote up
public void testDataBlockEncoding(DataBlockEncoding encoding) throws Exception {
  String TN = "testDataBlockEncoding" + encoding;
  setUp(defaultThreshold, TN, encoding);
  long ts1 = System.currentTimeMillis();
  long ts2 = ts1 + 1;
  long ts3 = ts1 + 2;
  byte[] value = generateMobValue((int) defaultThreshold + 1);

  Put put1 = new Put(row1);
  put1.addColumn(family, qf1, ts3, value);
  put1.addColumn(family, qf2, ts2, value);
  put1.addColumn(family, qf3, ts1, value);
  table.put(put1);
  admin.flush(TableName.valueOf(TN));

  Scan scan = new Scan();
  scan.readVersions(4);
  MobTestUtil.assertCellsValue(table, scan, value, 3);
}
 
Example #12
Source File: HFileOutputFormat3.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Serialize column family to data block encoding map to configuration.
 * Invoked while configuring the MR job for incremental load.
 *
 * @param table to read the properties from
 * @param conf to persist serialized values into
 * @throws IOException
 *           on failure to read column family descriptors
 */
@VisibleForTesting
static void configureDataBlockEncoding(HTableDescriptor tableDescriptor, Configuration conf)
        throws UnsupportedEncodingException {
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;
    }
    StringBuilder dataBlockEncodingConfigValue = new StringBuilder();
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            dataBlockEncodingConfigValue.append('&');
        }
        dataBlockEncodingConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        dataBlockEncodingConfigValue.append('=');
        DataBlockEncoding encoding = familyDescriptor.getDataBlockEncoding();
        if (encoding == null) {
            encoding = DataBlockEncoding.NONE;
        }
        dataBlockEncodingConfigValue.append(URLEncoder.encode(encoding.toString(), "UTF-8"));
    }
    conf.set(DATABLOCK_ENCODING_FAMILIES_CONF_KEY, dataBlockEncodingConfigValue.toString());
}
 
Example #13
Source File: TestHFileOutputFormat2.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @return a map from column family names to compression algorithms for
 *         testing column family compression. Column family names have special characters
 */
private Map<String, DataBlockEncoding>
    getMockColumnFamiliesForDataBlockEncoding (int numCfs) {
  Map<String, DataBlockEncoding> familyToDataBlockEncoding = new HashMap<>();
  // use column family names having special characters
  if (numCfs-- > 0) {
    familyToDataBlockEncoding.put("Family1!@#!@#&", DataBlockEncoding.DIFF);
  }
  if (numCfs-- > 0) {
    familyToDataBlockEncoding.put("Family2=asdads&!AASD",
        DataBlockEncoding.FAST_DIFF);
  }
  if (numCfs-- > 0) {
    familyToDataBlockEncoding.put("Family2=asdads&!AASD",
        DataBlockEncoding.PREFIX);
  }
  if (numCfs-- > 0) {
    familyToDataBlockEncoding.put("Family3", DataBlockEncoding.NONE);
  }
  return familyToDataBlockEncoding;
}
 
Example #14
Source File: TestBulkLoadHFiles.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void testSplitStoreFileWithDifferentEncoding(DataBlockEncoding bulkloadEncoding,
    DataBlockEncoding cfEncoding) throws IOException {
  Path dir = util.getDataTestDirOnTestFS("testSplitHFileWithDifferentEncoding");
  FileSystem fs = util.getTestFileSystem();
  Path testIn = new Path(dir, "testhfile");
  ColumnFamilyDescriptor familyDesc =
    ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setDataBlockEncoding(cfEncoding).build();
  HFileTestUtil.createHFileWithDataBlockEncoding(util.getConfiguration(), fs, testIn,
    bulkloadEncoding, FAMILY, QUALIFIER, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);

  Path bottomOut = new Path(dir, "bottom.out");
  Path topOut = new Path(dir, "top.out");

  BulkLoadHFilesTool.splitStoreFile(util.getConfiguration(), testIn, familyDesc,
    Bytes.toBytes("ggg"), bottomOut, topOut);

  int rowCount = verifyHFile(bottomOut);
  rowCount += verifyHFile(topOut);
  assertEquals(1000, rowCount);
}
 
Example #15
Source File: HBaseTestingUtility.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Create a set of column descriptors with the combination of compression,
 * encoding, bloom codecs available.
 * @param prefix family names prefix
 * @return the list of column descriptors
 */
public static List<ColumnFamilyDescriptor> generateColumnDescriptors(final String prefix) {
  List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
  long familyId = 0;
  for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) {
    for (DataBlockEncoding encodingType: DataBlockEncoding.values()) {
      for (BloomType bloomType: BloomType.values()) {
        String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId);
        ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder =
          ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(name));
        columnFamilyDescriptorBuilder.setCompressionType(compressionType);
        columnFamilyDescriptorBuilder.setDataBlockEncoding(encodingType);
        columnFamilyDescriptorBuilder.setBloomFilterType(bloomType);
        columnFamilyDescriptors.add(columnFamilyDescriptorBuilder.build());
        familyId++;
      }
    }
  }
  return columnFamilyDescriptors;
}
 
Example #16
Source File: HBaseTestingUtility.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a pre-split table for load testing. If the table already exists,
 * logs a warning and continues.
 * @return the number of regions the table was split into
 */
public static int createPreSplitLoadTestTable(Configuration conf,
    TableName tableName, byte[] columnFamily, Algorithm compression,
    DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication,
    Durability durability)
        throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setDurability(durability);
  tableDescriptor.setRegionReplication(regionReplication);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(columnFamily);
  familyDescriptor.setDataBlockEncoding(dataBlockEncoding);
  familyDescriptor.setCompressionType(compression);
  return createPreSplitLoadTestTable(conf, tableDescriptor, familyDescriptor,
    numRegionsPerServer);
}
 
Example #17
Source File: DataBlockEncodingValidator.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Check DataBlockEncodings of column families are compatible.
 *
 * @return number of column families with incompatible DataBlockEncoding
 * @throws IOException if a remote or network exception occurs
 */
private int validateDBE() throws IOException {
  int incompatibilities = 0;

  LOG.info("Validating Data Block Encodings");

  try (Connection connection = ConnectionFactory.createConnection(getConf());
      Admin admin = connection.getAdmin()) {
    List<TableDescriptor> tableDescriptors = admin.listTableDescriptors();
    String encoding = "";

    for (TableDescriptor td : tableDescriptors) {
      ColumnFamilyDescriptor[] columnFamilies = td.getColumnFamilies();
      for (ColumnFamilyDescriptor cfd : columnFamilies) {
        try {
          encoding = Bytes.toString(cfd.getValue(DATA_BLOCK_ENCODING));
          // IllegalArgumentException will be thrown if encoding is incompatible with 2.0
          DataBlockEncoding.valueOf(encoding);
        } catch (IllegalArgumentException e) {
          incompatibilities++;
          LOG.warn("Incompatible DataBlockEncoding for table: {}, cf: {}, encoding: {}",
              td.getTableName().getNameAsString(), cfd.getNameAsString(), encoding);
        }
      }
    }
  }

  if (incompatibilities > 0) {
    LOG.warn("There are {} column families with incompatible Data Block Encodings. Do not "
        + "upgrade until these encodings are converted to a supported one. "
        + "Check https://s.apache.org/prefixtree for instructions.", incompatibilities);
  } else {
    LOG.info("The used Data Block Encodings are compatible with HBase 2.0.");
  }

  return incompatibilities;
}
 
Example #18
Source File: HBaseStoreManager.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
private void setCFOptions(HColumnDescriptor cdesc, int ttlInSeconds) {
    if (null != compression && !compression.equals(COMPRESSION_DEFAULT))
        compat.setCompression(cdesc, compression);

    if (ttlInSeconds > 0)
        cdesc.setTimeToLive(ttlInSeconds);

    cdesc.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
}
 
Example #19
Source File: HFileOutputFormat3.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
/**
 * Runs inside the task to deserialize column family to data block encoding
 * type map from the configuration.
 *
 * @param conf to read the serialized values from
 * @return a map from column family to HFileDataBlockEncoder for the
 *         configured data block type for the family
 */
@VisibleForTesting
static Map<byte[], DataBlockEncoding> createFamilyDataBlockEncodingMap(Configuration conf) {
    Map<byte[], String> stringMap = createFamilyConfValueMap(conf, DATABLOCK_ENCODING_FAMILIES_CONF_KEY);
    Map<byte[], DataBlockEncoding> encoderMap = new TreeMap<byte[], DataBlockEncoding>(Bytes.BYTES_COMPARATOR);
    for (Map.Entry<byte[], String> e : stringMap.entrySet()) {
        encoderMap.put(e.getKey(), DataBlockEncoding.valueOf((e.getValue())));
    }
    return encoderMap;
}
 
Example #20
Source File: MultiHfileOutputFormat.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Runs inside the task to deserialize column family to data block encoding
 * type map from the configuration.
 *
 * @param conf to read the serialized values from
 * @return a map from column family to HFileDataBlockEncoder for the
 *         configured data block type for the family
 */
@VisibleForTesting
static Map<byte[], DataBlockEncoding> createFamilyDataBlockEncodingMap(Configuration conf,final String tableName) {
    
    Map<byte[], DataBlockEncoding> encoderMap = new TreeMap<byte[],DataBlockEncoding>(Bytes.BYTES_COMPARATOR);
    Map<String, String> tableConfigs = getTableConfigurations(conf, tableName);
    if(tableConfigs == null) {
        return encoderMap;
    }
    Map<byte[], String> stringMap = createFamilyConfValueMap(tableConfigs,DATABLOCK_ENCODING_FAMILIES_CONF_KEY);
    for (Map.Entry<byte[], String> e : stringMap.entrySet()) {
        encoderMap.put(e.getKey(), DataBlockEncoding.valueOf((e.getValue())));
    }
    return encoderMap;
}
 
Example #21
Source File: EncodedSeekPerformanceTest.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Command line interface:
 * @param args Takes one argument - file size.
 * @throws IOException if there is a bug while reading from disk
 */
public static void main(final String[] args) throws IOException {
  if (args.length < 1) {
    printUsage();
    System.exit(-1);
  }

  Path path = new Path(args[0]);

  // TODO, this test doesn't work as expected any more. Need to fix.
  EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest();
  utility.runTests(path, DataBlockEncoding.values());

  System.exit(0);
}
 
Example #22
Source File: TestHFileOutputFormat2.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test for {@link HFileOutputFormat2#createFamilyDataBlockEncodingMap(Configuration)}.
 * Tests that the family data block encoding map is correctly serialized into
 * and deserialized from configuration
 *
 * @throws IOException
 */
@Ignore("Goes zombie too frequently; needs work. See HBASE-14563") @Test
public void testSerializeDeserializeFamilyDataBlockEncodingMap() throws IOException {
  for (int numCfs = 0; numCfs <= 3; numCfs++) {
    Configuration conf = new Configuration(this.util.getConfiguration());
    Map<String, DataBlockEncoding> familyToDataBlockEncoding =
        getMockColumnFamiliesForDataBlockEncoding(numCfs);
    Table table = Mockito.mock(Table.class);
    setupMockColumnFamiliesForDataBlockEncoding(table,
        familyToDataBlockEncoding);
    TableDescriptor tableDescriptor = table.getDescriptor();
    conf.set(HFileOutputFormat2.DATABLOCK_ENCODING_FAMILIES_CONF_KEY,
            HFileOutputFormat2.serializeColumnFamilyAttribute
                    (HFileOutputFormat2.dataBlockEncodingDetails, Arrays
                    .asList(tableDescriptor)));

    // read back family specific data block encoding settings from the
    // configuration
    Map<byte[], DataBlockEncoding> retrievedFamilyToDataBlockEncodingMap =
        HFileOutputFormat2
        .createFamilyDataBlockEncodingMap(conf);

    // test that we have a value for all column families that matches with the
    // used mock values
    for (Entry<String, DataBlockEncoding> entry : familyToDataBlockEncoding.entrySet()) {
      assertEquals("DataBlockEncoding configuration incorrect for column family:"
          + entry.getKey(), entry.getValue(),
          retrievedFamilyToDataBlockEncodingMap.get(Bytes.toBytes(entry.getKey())));
    }
  }
}
 
Example #23
Source File: HBaseTestingUtility.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a pre-split table for load testing. If the table already exists,
 * logs a warning and continues.
 * @return the number of regions the table was split into
 */
public static int createPreSplitLoadTestTable(Configuration conf,
    TableName tableName, byte[] columnFamily, Algorithm compression,
    DataBlockEncoding dataBlockEncoding) throws IOException {
  return createPreSplitLoadTestTable(conf, tableName,
    columnFamily, compression, dataBlockEncoding, DEFAULT_REGIONS_PER_SERVER, 1,
    Durability.USE_DEFAULT);
}
 
Example #24
Source File: HFileReaderImpl.java    From hbase with Apache License 2.0 5 votes vote down vote up
public EncodedScanner(HFile.Reader reader, boolean cacheBlocks,
    boolean pread, boolean isCompaction, HFileContext meta) {
  super(reader, cacheBlocks, pread, isCompaction);
  DataBlockEncoding encoding = reader.getDataBlockEncoding();
  dataBlockEncoder = encoding.getEncoder();
  decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
  seeker = dataBlockEncoder.createSeeker(decodingCtx);
}
 
Example #25
Source File: TestHFileWriterV3WithDataEncoders.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> parameters() {
  DataBlockEncoding[] dataBlockEncodings = DataBlockEncoding.values();
  Object[][] params = new Object[dataBlockEncodings.length * 2 - 2][];
  int i = 0;
  for (DataBlockEncoding dataBlockEncoding : dataBlockEncodings) {
    if (dataBlockEncoding == DataBlockEncoding.NONE) {
      continue;
    }
    params[i++] = new Object[]{false, dataBlockEncoding};
    params[i++] = new Object[]{true, dataBlockEncoding};
  }
  return Arrays.asList(params);
}
 
Example #26
Source File: TestHStore.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Verify that compression and data block encoding are respected by the
 * Store.createWriterInTmp() method, used on store flush.
 */
@Test
public void testCreateWriter() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  FileSystem fs = FileSystem.get(conf);

  ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family)
      .setCompressionType(Compression.Algorithm.GZ).setDataBlockEncoding(DataBlockEncoding.DIFF)
      .build();
  init(name.getMethodName(), conf, hcd);

  // Test createWriterInTmp()
  StoreFileWriter writer =
      store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false);
  Path path = writer.getPath();
  writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
  writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
  writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
  writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
  writer.close();

  // Verify that compression and encoding settings are respected
  HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf);
  assertEquals(hcd.getCompressionType(), reader.getTrailer().getCompressionCodec());
  assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
  reader.close();
}
 
Example #27
Source File: TestStoreFileScannerWithTagCompression.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testReseek() throws Exception {
  // write the file
  Path f = new Path(ROOT_DIR, "testReseek");
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).withIncludesTags(true)
      .withCompressTags(true).withDataBlockEncoding(DataBlockEncoding.PREFIX).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(f)
      .withFileContext(meta).build();

  writeStoreFile(writer);
  writer.close();

  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
  HFileInfo fileInfo = new HFileInfo(context, conf);
  StoreFileReader reader =
      new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
  fileInfo.initMetaAndIndex(reader.getHFileReader());
  StoreFileScanner s = reader.getStoreFileScanner(false, false, false, 0, 0, false);
  try {
    // Now do reseek with empty KV to position to the beginning of the file
    KeyValue k = KeyValueUtil.createFirstOnRow(Bytes.toBytes("k2"));
    s.reseek(k);
    Cell kv = s.next();
    kv = s.next();
    kv = s.next();
    byte[] key5 = Bytes.toBytes("k5");
    assertTrue(Bytes.equals(key5, 0, key5.length, kv.getRowArray(), kv.getRowOffset(),
        kv.getRowLength()));
    List<Tag> tags = PrivateCellUtil.getTags(kv);
    assertEquals(1, tags.size());
    assertEquals("tag3", Bytes.toString(Tag.cloneValue(tags.get(0))));
  } finally {
    s.close();
  }
}
 
Example #28
Source File: QueryDatabaseMetaDataIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private static void createMDTestTable(Connection conn, String tableName, String extraProps)
        throws SQLException {
    String ddl =
            "create table if not exists " + tableName + "   (id char(1) primary key,\n"
                    + "    a.col1 integer,\n" + "    b.col2 bigint,\n" + "    b.col3 decimal,\n"
                    + "    b.col4 decimal(5),\n" + "    b.col5 decimal(6,3))\n" + "    a."
                    + HConstants.VERSIONS + "=" + 1 + "," + "a."
                    + ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING + "='" + DataBlockEncoding.NONE
                    + "'";
    if (extraProps != null && extraProps.length() > 0) {
        ddl += "," + extraProps;
    }
    conn.createStatement().execute(ddl);
}
 
Example #29
Source File: HFileTestUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Create an HFile with the given number of rows between a given
 * start key and end key @ family:qualifier.
 * If withTag is true, we add the rowKey as the tag value for
 * tagtype MOB_TABLE_NAME_TAG_TYPE
 */
public static void createHFile(
    Configuration configuration,
    FileSystem fs, Path path, DataBlockEncoding encoding,
    byte[] family, byte[] qualifier,
    byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException {
  HFileContext meta = new HFileContextBuilder()
      .withIncludesTags(withTag)
      .withDataBlockEncoding(encoding)
      .withColumnFamily(family)
      .build();
  HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration))
      .withPath(fs, path)
      .withFileContext(meta)
      .create();
  long now = System.currentTimeMillis();
  try {
    // subtract 2 since iterateOnSplits doesn't include boundary keys
    for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) {
      Cell kv = new KeyValue(key, family, qualifier, now, key);
      if (withTag) {
        // add a tag.  Arbitrarily chose mob tag since we have a helper already.
        Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, key);
        kv = MobUtils.createMobRefCell(kv, key, tableNameTag);

        // verify that the kv has the tag.
        Optional<Tag> tag = PrivateCellUtil.getTag(kv, TagType.MOB_TABLE_NAME_TAG_TYPE);
        if (!tag.isPresent()) {
          throw new IllegalStateException("Tag didn't stick to KV " + kv.toString());
        }
      }
      writer.append(kv);
    }
  } finally {
    writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
    writer.close();
  }
}
 
Example #30
Source File: TestHFileDataBlockEncoder.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void testEncodingInternals(boolean useTag) throws IOException {
  // usually we have just block without headers, but don't complicate that
  List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
  HFileBlock block = getSampleHFileBlock(kvs, useTag);
  HFileBlock blockOnDisk = createBlockOnDisk(kvs, block, useTag);

  if (blockEncoder.getDataBlockEncoding() !=
      DataBlockEncoding.NONE) {
    assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
    assertEquals(blockEncoder.getDataBlockEncoding().getId(),
        blockOnDisk.getDataBlockEncodingId());
  } else {
    assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
  }
}