Java Code Examples for org.apache.hadoop.hbase.io.compress.Compression#Algorithm

private void setupMockColumnFamiliesForCompression(Table table,
    Map<String, Compression.Algorithm> familyToCompression) throws IOException {

  TableDescriptorBuilder mockTableDescriptor =
    TableDescriptorBuilder.newBuilder(TABLE_NAMES[0]);
  for (Entry<String, Compression.Algorithm> entry : familyToCompression.entrySet()) {
    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(entry.getKey()))
      .setMaxVersions(1)
      .setCompressionType(entry.getValue())
      .setBlockCacheEnabled(false)
      .setTimeToLive(0)
      .build();

    mockTableDescriptor.setColumnFamily(columnFamilyDescriptor);
  }
  Mockito.doReturn(mockTableDescriptor.build()).when(table).getDescriptor();
}
 

@Test
public void testNullMetaBlocks() throws Exception {
  for (Compression.Algorithm compressAlgo :
      HBaseCommonTestingUtility.COMPRESSION_ALGORITHMS) {
    Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
    FSDataOutputStream fout = createFSOutput(mFile);
    HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
                        .withBlockSize(minBlockSize).build();
    Writer writer = HFile.getWriterFactory(conf, cacheConf)
        .withOutputStream(fout)
        .withFileContext(meta)
        .create();
    KeyValue kv = new KeyValue(Bytes.toBytes("foo"), Bytes.toBytes("f1"), null,
        Bytes.toBytes("value"));
    writer.append(kv);
    writer.close();
    fout.close();
    Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf);
    assertNull(reader.getMetaBlock("non-existant", false));
  }
}
 

static void assertBuffersEqual(ByteBuff expectedBuffer,
    ByteBuff actualBuffer, Compression.Algorithm compression,
    DataBlockEncoding encoding, boolean pread) {
  if (!actualBuffer.equals(expectedBuffer)) {
    int prefix = 0;
    int minLimit = Math.min(expectedBuffer.limit(), actualBuffer.limit());
    while (prefix < minLimit &&
        expectedBuffer.get(prefix) == actualBuffer.get(prefix)) {
      prefix++;
    }

    fail(String.format(
        "Content mismatch for %s, commonPrefix %d, expected %s, got %s",
        buildMessageDetails(compression, encoding, pread), prefix,
        nextBytesToStr(expectedBuffer, prefix),
        nextBytesToStr(actualBuffer, prefix)));
  }
}
 

private HFileContext createFileContext(Compression.Algorithm compression,
    boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
  if (compression == null) {
    compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
  }
  HFileContext hFileContext = new HFileContextBuilder()
                              .withIncludesMvcc(includeMVCCReadpoint)
                              .withIncludesTags(includesTag)
                              .withCompression(compression)
                              .withCompressTags(family.isCompressTags())
                              .withChecksumType(checksumType)
                              .withBytesPerCheckSum(bytesPerChecksum)
                              .withBlockSize(blocksize)
                              .withHBaseCheckSum(true)
                              .withDataBlockEncoding(family.getDataBlockEncoding())
                              .withEncryptionContext(cryptoContext)
                              .withCreateTime(EnvironmentEdgeManager.currentTime())
                              .withColumnFamily(family.getName())
                              .withTableName(region.getTableDescriptor()
                                  .getTableName().getName())
                              .withCellComparator(this.comparator)
                              .build();
  return hFileContext;
}
 

public String createTestBlockStr(Compression.Algorithm algo,
    int correctLength, boolean useTag) throws IOException {
  HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS, useTag);
  byte[] testV2Block = hbw.getHeaderAndDataForTest();
  int osOffset = HConstants.HFILEBLOCK_HEADER_SIZE + 9;
  if (testV2Block.length == correctLength) {
    // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
    // variations across operating systems.
    // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
    // We only make this change when the compressed block length matches.
    // Otherwise, there are obviously other inconsistencies.
    testV2Block[osOffset] = 3;
  }
  return Bytes.toStringBinary(testV2Block);
}
 

public byte[] createTestV1Block(Compression.Algorithm algo)
    throws IOException {
  Compressor compressor = algo.getCompressor();
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  OutputStream os = algo.createCompressionStream(baos, compressor, 0);
  DataOutputStream dos = new DataOutputStream(os);
  BlockType.META.write(dos); // Let's make this a meta block.
  writeTestBlockContents(dos);
  dos.flush();
  algo.returnCompressor(compressor);
  return baos.toByteArray();
}
 

/**
 * @param encoding encoding used
 * @param headerBytes dummy header bytes
 * @param fileContext HFile meta data
 */
public HFileBlockDefaultEncodingContext(DataBlockEncoding encoding, byte[] headerBytes,
    HFileContext fileContext) {
  this.encodingAlgo = encoding;
  this.fileContext = fileContext;
  Compression.Algorithm compressionAlgorithm =
      fileContext.getCompression() == null ? NONE : fileContext.getCompression();
  if (compressionAlgorithm != NONE) {
    compressor = compressionAlgorithm.getCompressor();
    compressedByteStream = new ByteArrayOutputStream();
    try {
      compressionStream =
          compressionAlgorithm.createPlainCompressionStream(
              compressedByteStream, compressor);
    } catch (IOException e) {
      throw new RuntimeException(
          "Could not create compression stream for algorithm "
              + compressionAlgorithm, e);
    }
  }

  Encryption.Context cryptoContext = fileContext.getEncryptionContext();
  if (cryptoContext != Encryption.Context.NONE) {
    cryptoByteStream = new ByteArrayOutputStream();
    iv = new byte[cryptoContext.getCipher().getIvLength()];
    new SecureRandom().nextBytes(iv);
  }

  dummyHeader = Preconditions.checkNotNull(headerBytes,
    "Please pass HConstants.HFILEBLOCK_DUMMY_HEADER instead of null for param headerBytes");
}
 

private void testHFileFormatV3Internals(boolean useTags) throws IOException {
  Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV3");
  final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ;
  final int entryCount = 10000;
  writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false, useTags);
}
 

public ColumnFamilyDescriptorBuilder setCompressionType(Compression.Algorithm value) {
  desc.setCompressionType(value);
  return this;
}
 

private long writeBlocks(Random rand, Compression.Algorithm compressAlgo,
    Path path, List<Long> expectedOffsets, List<Long> expectedPrevOffsets,
    List<BlockType> expectedTypes, List<ByteBuffer> expectedContents
) throws IOException {
  boolean cacheOnWrite = expectedContents != null;
  FSDataOutputStream os = fs.create(path);
  HFileContext meta = new HFileContextBuilder()
                      .withHBaseCheckSum(true)
                      .withIncludesMvcc(includesMemstoreTS)
                      .withIncludesTags(includesTag)
                      .withCompression(compressAlgo)
                      .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
                      .build();
  HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
  Map<BlockType, Long> prevOffsetByType = new HashMap<>();
  long totalSize = 0;
  for (int i = 0; i < NUM_TEST_BLOCKS; ++i) {
    long pos = os.getPos();
    int blockTypeOrdinal = rand.nextInt(BlockType.values().length);
    if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) {
      blockTypeOrdinal = BlockType.DATA.ordinal();
    }
    BlockType bt = BlockType.values()[blockTypeOrdinal];
    DataOutputStream dos = hbw.startWriting(bt);
    int size = rand.nextInt(500);
    for (int j = 0; j < size; ++j) {
      // This might compress well.
      dos.writeShort(i + 1);
      dos.writeInt(j + 1);
    }

    if (expectedOffsets != null)
      expectedOffsets.add(os.getPos());

    if (expectedPrevOffsets != null) {
      Long prevOffset = prevOffsetByType.get(bt);
      expectedPrevOffsets.add(prevOffset != null ? prevOffset : -1);
      prevOffsetByType.put(bt, os.getPos());
    }

    expectedTypes.add(bt);

    hbw.writeHeaderAndData(os);
    totalSize += hbw.getOnDiskSizeWithHeader();

    if (cacheOnWrite) {
      ByteBuff buff = hbw.cloneUncompressedBufferWithHeader();
      expectedContents.add(buff.asSubByteBuffer(buff.capacity()));
    }

    if (detailedLogging) {
      LOG.info("Written block #" + i + " of type " + bt
          + ", uncompressed size " + hbw.getUncompressedSizeWithoutHeader()
          + ", packed size " + hbw.getOnDiskSizeWithoutHeader()
          + " at offset " + pos);
    }
  }
  os.close();
  LOG.info("Created a temporary file at " + path + ", "
      + fs.getFileStatus(path).getLen() + " byte, compression=" +
      compressAlgo);
  return totalSize;
}
 

public ColumnFamilyDescriptorBuilder setCompactionCompressionType(Compression.Algorithm value) {
  desc.setCompactionCompressionType(value);
  return this;
}
 

@VisibleForTesting
public Compression.Algorithm getCompressionAlgorithm() {
  return trailer.getCompressionCodec();
}
 

@Override
public Compression.Algorithm getCompactionCompressionType() {
  return delegatee.getCompactionCompressionType();
}
 

/**
 * @param compression Compression algorithm to use
 * @param isCompaction whether we are creating a new file in a compaction
 * @param includeMVCCReadpoint - whether to include MVCC or not
 * @param includesTag - includesTag or not
 * @return Writer for a new StoreFile in the tmp dir.
 */
// TODO : allow the Writer factory to create Writers of ShipperListener type only in case of
// compaction
public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
    boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,
    boolean shouldDropBehind, long totalCompactedFilesSize, String fileStoragePolicy)
      throws IOException {
  // creating new cache config for each new writer
  final CacheConfig writerCacheConf = new CacheConfig(cacheConf);
  if (isCompaction) {
    // Don't cache data on write on compactions, unless specifically configured to do so
    // Cache only when total file size remains lower than configured threshold
    final boolean cacheCompactedBlocksOnWrite =
      cacheConf.shouldCacheCompactedBlocksOnWrite();
    // if data blocks are to be cached on write
    // during compaction, we should forcefully
    // cache index and bloom blocks as well
    if (cacheCompactedBlocksOnWrite && totalCompactedFilesSize <= cacheConf
      .getCacheCompactedBlocksOnWriteThreshold()) {
      writerCacheConf.enableCacheOnWrite();
      if (!cacheOnWriteLogged) {
        LOG.info("For {} , cacheCompactedBlocksOnWrite is true, hence enabled " +
            "cacheOnWrite for Data blocks, Index blocks and Bloom filter blocks", this);
        cacheOnWriteLogged = true;
      }
    } else {
      writerCacheConf.setCacheDataOnWrite(false);
      if (totalCompactedFilesSize > cacheConf.getCacheCompactedBlocksOnWriteThreshold()) {
        // checking condition once again for logging
        LOG.debug(
          "For {}, setting cacheCompactedBlocksOnWrite as false as total size of compacted "
            + "files - {}, is greater than cacheCompactedBlocksOnWriteThreshold - {}",
          this, totalCompactedFilesSize,
          cacheConf.getCacheCompactedBlocksOnWriteThreshold());
      }
    }
  } else {
    final boolean shouldCacheDataOnWrite = cacheConf.shouldCacheDataOnWrite();
    if (shouldCacheDataOnWrite) {
      writerCacheConf.enableCacheOnWrite();
      if (!cacheOnWriteLogged) {
        LOG.info("For {} , cacheDataOnWrite is true, hence enabled cacheOnWrite for " +
          "Index blocks and Bloom filter blocks", this);
        cacheOnWriteLogged = true;
      }
    }
  }
  InetSocketAddress[] favoredNodes = null;
  if (region.getRegionServerServices() != null) {
    favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
        region.getRegionInfo().getEncodedName());
  }
  HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
    cryptoContext);
  Path familyTempDir = new Path(fs.getTempDir(), family.getNameAsString());
  StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
      this.getFileSystem())
          .withOutputDir(familyTempDir)
          .withBloomType(family.getBloomFilterType())
          .withMaxKeyCount(maxKeyCount)
          .withFavoredNodes(favoredNodes)
          .withFileContext(hFileContext)
          .withShouldDropCacheBehind(shouldDropBehind)
          .withCompactedFilesSupplier(this::getCompactedFiles)
          .withFileStoragePolicy(fileStoragePolicy);
  return builder.build();
}
 

public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
  boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,
  boolean shouldDropBehind) throws IOException {
  return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint,
    includesTag, shouldDropBehind, -1, HConstants.EMPTY_STRING);
}
 

public void setCompression(Compression.Algorithm compression) {
  this.compression = compression;
}
 

public Compression.Algorithm getCompression() {
  return compression;
}
 

@Test
public void testDataBlockEncryption() throws IOException {
  final int blocks = 10;
  final int[] blockSizes = new int[blocks];
  for (int i = 0; i < blocks; i++) {
    blockSizes[i] = (1024 + RNG.nextInt(1024 * 63)) / Bytes.SIZEOF_INT;
  }
  for (Compression.Algorithm compression : TestHFileBlock.COMPRESSION_ALGORITHMS) {
    Path path = new Path(TEST_UTIL.getDataTestDir(), "block_v3_" + compression + "_AES");
    LOG.info("testDataBlockEncryption: encryption=AES compression=" + compression);
    long totalSize = 0;
    HFileContext fileContext = new HFileContextBuilder()
      .withCompression(compression)
      .withEncryptionContext(cryptoContext)
      .build();
    FSDataOutputStream os = fs.create(path);
    try {
      for (int i = 0; i < blocks; i++) {
        totalSize += writeBlock(os, fileContext, blockSizes[i]);
      }
    } finally {
      os.close();
    }
    FSDataInputStream is = fs.open(path);
    ReaderContext context = new ReaderContextBuilder()
        .withInputStreamWrapper(new FSDataInputStreamWrapper(is))
        .withFilePath(path)
        .withFileSystem(fs)
        .withFileSize(totalSize).build();
    try {
      HFileBlock.FSReaderImpl hbr = new HFileBlock.FSReaderImpl(context, fileContext,
          ByteBuffAllocator.HEAP);
      long pos = 0;
      for (int i = 0; i < blocks; i++) {
        pos += readAndVerifyBlock(pos, fileContext, hbr, blockSizes[i]);
      }
    } finally {
      is.close();
    }
  }
}
 

/**
 * Creates a writer for the mob file in temp directory.
 * @param conf The current configuration.
 * @param fs The current file system.
 * @param family The descriptor of the current column family.
 * @param mobFileName The mob file name.
 * @param basePath The basic path for a temp directory.
 * @param maxKeyCount The key count.
 * @param compression The compression algorithm.
 * @param cacheConfig The current cache config.
 * @param cryptoContext The encryption context.
 * @param isCompaction If the writer is used in compaction.
 * @return The writer for the mob file.
 */
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
    ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
    Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext,
    boolean isCompaction) throws IOException {
  return createWriter(conf, fs, family, new Path(basePath, mobFileName.getFileName()),
    maxKeyCount, compression, cacheConfig, cryptoContext, HStore.getChecksumType(conf),
    HStore.getBytesPerChecksum(conf), family.getBlocksize(), BloomType.NONE, isCompaction);
}
 

/**
 * Creates a writer for the mob file in temp directory.
 * @param conf The current configuration.
 * @param fs The current file system.
 * @param family The descriptor of the current column family.
 * @param date The date string, its format is yyyymmmdd.
 * @param basePath The basic path for a temp directory.
 * @param maxKeyCount The key count.
 * @param compression The compression algorithm.
 * @param startKey The hex string of the start key.
 * @param cacheConfig The current cache config.
 * @param cryptoContext The encryption context.
 * @param isCompaction If the writer is used in compaction.
 * @return The writer for the mob file.
 */
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
    ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
    Compression.Algorithm compression, String startKey, CacheConfig cacheConfig,
    Encryption.Context cryptoContext, boolean isCompaction, String regionName)
    throws IOException {
  MobFileName mobFileName = MobFileName.create(startKey, date,
    UUID.randomUUID().toString().replaceAll("-", ""), regionName);
  return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
    cacheConfig, cryptoContext, isCompaction);
}