Java Code Examples for org.apache.hadoop.hbase.io.hfile.HFileContext

The following examples show how to use org.apache.hadoop.hbase.io.hfile.HFileContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: HStore.java    License: Apache License 2.0 6 votes vote down vote up
private HFileContext createFileContext(Compression.Algorithm compression,
    boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
  if (compression == null) {
    compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
  }
  HFileContext hFileContext = new HFileContextBuilder()
                              .withIncludesMvcc(includeMVCCReadpoint)
                              .withIncludesTags(includesTag)
                              .withCompression(compression)
                              .withCompressTags(family.isCompressTags())
                              .withChecksumType(checksumType)
                              .withBytesPerCheckSum(bytesPerChecksum)
                              .withBlockSize(blocksize)
                              .withHBaseCheckSum(true)
                              .withDataBlockEncoding(family.getDataBlockEncoding())
                              .withEncryptionContext(cryptoContext)
                              .withCreateTime(EnvironmentEdgeManager.currentTime())
                              .withColumnFamily(family.getName())
                              .withTableName(region.getTableDescriptor()
                                  .getTableName().getName())
                              .withCellComparator(this.comparator)
                              .build();
  return hFileContext;
}
 
Example 2
Source Project: hbase   Source File: TestAccessController.java    License: Apache License 2.0 6 votes vote down vote up
private void createHFile(Path path,
    byte[] family, byte[] qualifier,
    byte[] startKey, byte[] endKey, int numRows) throws IOException {
  HFile.Writer writer = null;
  long now = System.currentTimeMillis();
  try {
    HFileContext context = new HFileContextBuilder().build();
    writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
        .withFileContext(context).create();
    // subtract 2 since numRows doesn't include boundary keys
    for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, true, numRows - 2)) {
      KeyValue kv = new KeyValue(key, family, qualifier, now, key);
      writer.append(kv);
    }
  } finally {
    if (writer != null) {
      writer.close();
    }
  }
}
 
Example 3
Source Project: hbase   Source File: TestMobFile.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetScanner() throws Exception {
  Path testDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(conf);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs)
          .withOutputDir(testDir)
          .withFileContext(meta)
          .build();
  MobTestUtil.writeStoreFile(writer, testName.getMethodName());

  MobFile mobFile =
      new MobFile(new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true));
  assertNotNull(mobFile.getScanner());
  assertTrue(mobFile.getScanner() instanceof StoreFileScanner);
}
 
Example 4
Source Project: hbase   Source File: TestCachedMobFile.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOpenClose() throws Exception {
  String caseName = testName.getMethodName();
  Path testDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(conf);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs)
      .withOutputDir(testDir).withFileContext(meta).build();
  MobTestUtil.writeStoreFile(writer, caseName);
  CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
  assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
  cachedMobFile.open();
  assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
  cachedMobFile.open();
  assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile.getReferenceCount());
  cachedMobFile.close();
  assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
  cachedMobFile.close();
  assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
}
 
Example 5
Source Project: hbase   Source File: TestCachedMobFile.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("SelfComparison")
@Test
public void testCompare() throws Exception {
  String caseName = testName.getMethodName();
  Path testDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(conf);
  Path outputDir1 = new Path(testDir, FAMILY1);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  StoreFileWriter writer1 = new StoreFileWriter.Builder(conf, cacheConf, fs)
      .withOutputDir(outputDir1).withFileContext(meta).build();
  MobTestUtil.writeStoreFile(writer1, caseName);
  CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf);
  Path outputDir2 = new Path(testDir, FAMILY2);
  StoreFileWriter writer2 = new StoreFileWriter.Builder(conf, cacheConf, fs)
      .withOutputDir(outputDir2)
      .withFileContext(meta)
      .build();
  MobTestUtil.writeStoreFile(writer2, caseName);
  CachedMobFile cachedMobFile2 = CachedMobFile.create(fs, writer2.getPath(), conf, cacheConf);
  cachedMobFile1.access(1);
  cachedMobFile2.access(2);
  assertEquals(1, cachedMobFile1.compareTo(cachedMobFile2));
  assertEquals(-1, cachedMobFile2.compareTo(cachedMobFile1));
  assertEquals(0, cachedMobFile1.compareTo(cachedMobFile1));
}
 
Example 6
Source Project: hbase   Source File: HFilePerformanceEvaluation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void setUp() throws Exception {

  HFileContextBuilder builder = new HFileContextBuilder()
      .withCompression(HFileWriterImpl.compressionByName(codec))
      .withBlockSize(RFILE_BLOCKSIZE);
  
  if (cipher == "aes") {
    byte[] cipherKey = new byte[AES.KEY_LENGTH];
    new SecureRandom().nextBytes(cipherKey);
    builder.withEncryptionContext(Encryption.newContext(conf)
        .setCipher(Encryption.getCipher(conf, cipher))
        .setKey(cipherKey));
  } else if (!"none".equals(cipher)) {
    throw new IOException("Cipher " + cipher + " not supported.");
  }
  
  HFileContext hFileContext = builder.build();

  writer = HFile.getWriterFactoryNoCache(conf)
      .withPath(fs, mf)
      .withFileContext(hFileContext)
      .create();
}
 
Example 7
Source Project: hbase   Source File: TestRegionObserverInterface.java    License: Apache License 2.0 6 votes vote down vote up
private static void createHFile(Configuration conf, FileSystem fs, Path path, byte[] family,
    byte[] qualifier) throws IOException {
  HFileContext context = new HFileContextBuilder().build();
  HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
      .withFileContext(context).create();
  long now = System.currentTimeMillis();
  try {
    for (int i = 1; i <= 9; i++) {
      KeyValue kv =
          new KeyValue(Bytes.toBytes(i + ""), family, qualifier, now, Bytes.toBytes(i + ""));
      writer.append(kv);
    }
  } finally {
    writer.close();
  }
}
 
Example 8
Source Project: hbase   Source File: TestHStore.java    License: Apache License 2.0 6 votes vote down vote up
private void addStoreFile() throws IOException {
  HStoreFile f = this.store.getStorefiles().iterator().next();
  Path storedir = f.getPath().getParent();
  long seqid = this.store.getMaxSequenceId().orElse(0L);
  Configuration c = TEST_UTIL.getConfiguration();
  FileSystem fs = FileSystem.get(c);
  HFileContext fileContext = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
  StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c),
      fs)
          .withOutputDir(storedir)
          .withFileContext(fileContext)
          .build();
  w.appendMetadata(seqid + 1, false);
  w.close();
  LOG.info("Added store file:" + w.getPath());
}
 
Example 9
Source Project: hbase   Source File: TestStoreScannerClosure.java    License: Apache License 2.0 6 votes vote down vote up
private Path writeStoreFile() throws IOException {
  Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
  HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
  StoreFileWriter sfw = new StoreFileWriter.Builder(CONF, fs).withOutputDir(storeFileParentDir)
      .withFileContext(meta).build();

  final int rowLen = 32;
  Random RNG = new Random();
  for (int i = 0; i < 1000; ++i) {
    byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
    byte[] v = RandomKeyValueUtil.randomValue(RNG);
    int cfLen = RNG.nextInt(k.length - rowLen + 1);
    KeyValue kv = new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen,
        k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length);
    sfw.append(kv);
  }

  sfw.close();
  return sfw.getPath();
}
 
Example 10
Source Project: hbase   Source File: TestHRegionServerBulkLoad.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create an HFile with the given number of rows with a specified value.
 */
public static void createHFile(FileSystem fs, Path path, byte[] family,
    byte[] qualifier, byte[] value, int numRows) throws IOException {
  HFileContext context = new HFileContextBuilder().withBlockSize(BLOCKSIZE)
                          .withCompression(COMPRESSION)
                          .build();
  HFile.Writer writer = HFile
      .getWriterFactory(conf, new CacheConfig(conf))
      .withPath(fs, path)
      .withFileContext(context)
      .create();
  long now = System.currentTimeMillis();
  try {
    // subtract 2 since iterateOnSplits doesn't include boundary keys
    for (int i = 0; i < numRows; i++) {
      KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value);
      writer.append(kv);
    }
    writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(now));
  } finally {
    writer.close();
  }
}
 
Example 11
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write a file and then assert that we can read from top and bottom halves using two
 * HalfMapFiles.
 */
@Test
public void testBasicHalfMapFile() throws Exception {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testBasicHalfMapFileTb")).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
    new Path(testDir, hri.getTable().getNameAsString()), hri);

  HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);

  Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  HStoreFile sf = new HStoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE, true);
  checkHalfHFile(regionFs, sf);
}
 
Example 12
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Check if data block encoding information is saved correctly in HFile's file info.
 */
@Test
public void testDataBlockEncodingMetaData() throws IOException {
  // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.
  Path dir = new Path(new Path(testDir, "7e0102"), "familyname");
  Path path = new Path(dir, "1234567890");

  DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF;
  cacheConf = new CacheConfig(conf);
  HFileContext meta =
    new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE)
      .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();
  writer.close();

  HStoreFile storeFile =
    new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);
  storeFile.initReader();
  StoreFileReader reader = storeFile.getReader();

  Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
  byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);
  assertArrayEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
}
 
Example 13
Source Project: hbase   Source File: TestHeapSize.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHFileBlockSize() throws IOException {
  long expected;
  long actual;

  actual = HFileContext.FIXED_OVERHEAD;
  expected = ClassSize.estimateBase(HFileContext.class, false);
  assertEquals(expected, actual);

  actual = HFileBlock.FIXED_OVERHEAD;
  expected = ClassSize.estimateBase(HFileBlock.class, false);
  assertEquals(expected, actual);

  actual = ExclusiveMemHFileBlock.FIXED_OVERHEAD;
  expected = ClassSize.estimateBase(ExclusiveMemHFileBlock.class, false);
  assertEquals(expected, actual);

  actual = SharedMemHFileBlock.FIXED_OVERHEAD;
  expected = ClassSize.estimateBase(SharedMemHFileBlock.class, false);
  assertEquals(expected, actual);
}
 
Example 14
Source Project: hbase   Source File: TestSeekToBlockWithEncoders.java    License: Apache License 2.0 6 votes vote down vote up
private void seekToTheKey(KeyValue expected, List<KeyValue> kvs, Cell toSeek)
    throws IOException {
  // create all seekers
  List<DataBlockEncoder.EncodedSeeker> encodedSeekers = new ArrayList<>();
  for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
    if (encoding.getEncoder() == null) {
      continue;
    }
    DataBlockEncoder encoder = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false)
        .withIncludesMvcc(false).withIncludesTags(false)
        .withCompression(Compression.Algorithm.NONE).build();
    HFileBlockEncodingContext encodingContext = encoder.newDataBlockEncodingContext(encoding,
        HFILEBLOCK_DUMMY_HEADER, meta);
    ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs,
        encodingContext, this.useOffheapData);
    DataBlockEncoder.EncodedSeeker seeker =
      encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
    seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
    encodedSeekers.add(seeker);
  }
  // test it!
  // try a few random seeks
  checkSeekingConsistency(encodedSeekers, toSeek, expected);
}
 
Example 15
Source Project: hbase   Source File: TestDataBlockEncoders.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRowIndexWithTagsButNoTagsInCell() throws IOException {
  List<KeyValue> kvList = new ArrayList<>();
  byte[] row = new byte[0];
  byte[] family = new byte[0];
  byte[] qualifier = new byte[0];
  byte[] value = new byte[0];
  KeyValue expectedKV = new KeyValue(row, family, qualifier, 1L, Type.Put, value);
  kvList.add(expectedKV);
  DataBlockEncoding encoding = DataBlockEncoding.ROW_INDEX_V1;
  DataBlockEncoder encoder = encoding.getEncoder();
  ByteBuffer encodedBuffer =
      encodeKeyValues(encoding, kvList, getEncodingContext(Algorithm.NONE, encoding), false);
  HFileContext meta =
      new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS)
          .withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build();
  DataBlockEncoder.EncodedSeeker seeker =
    encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
  seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
  Cell cell = seeker.getCell();
  Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength());
}
 
Example 16
Source Project: hbase   Source File: TestDataBlockEncoders.java    License: Apache License 2.0 6 votes vote down vote up
private void testAlgorithm(byte[] encodedData, ByteBuffer unencodedDataBuf,
    DataBlockEncoder encoder) throws IOException {
  // decode
  ByteArrayInputStream bais = new ByteArrayInputStream(encodedData, ENCODED_DATA_OFFSET,
      encodedData.length - ENCODED_DATA_OFFSET);
  DataInputStream dis = new DataInputStream(bais);
  ByteBuffer actualDataset;
  HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false)
      .withIncludesMvcc(includesMemstoreTS).withIncludesTags(includesTags)
      .withCompression(Compression.Algorithm.NONE).build();
  actualDataset = encoder.decodeKeyValues(dis, encoder.newDataBlockDecodingContext(meta));
  actualDataset.rewind();

  // this is because in case of prefix tree the decoded stream will not have
  // the
  // mvcc in it.
  assertEquals("Encoding -> decoding gives different results for " + encoder,
      Bytes.toStringBinary(unencodedDataBuf), Bytes.toStringBinary(actualDataset));
}
 
Example 17
/**
 *
 * This is borrowed from DefaultCompactor.
 *
 * @param compression
 * @param includeMVCCReadpoint
 * @param includesTag
 * @param cryptoContext
 * @return
 */
private HFileContext createFileContext(Compression.Algorithm compression,
                                       boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
    if (compression == null) {
        compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
    }
    HFileContext hFileContext = new HFileContextBuilder()
            .withIncludesMvcc(includeMVCCReadpoint)
            .withIncludesTags(includesTag)
            .withCompression(compression)
            .withCompressTags(store.getColumnFamilyDescriptor().isCompressTags())
            .withChecksumType(HStore.getChecksumType(conf))
            .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
            .withBlockSize(store.getColumnFamilyDescriptor().getBlocksize())
            .withHBaseCheckSum(true)
            .withDataBlockEncoding(store.getColumnFamilyDescriptor().getDataBlockEncoding())
            .withEncryptionContext(cryptoContext)
            .withCreateTime(EnvironmentEdgeManager.currentTime())
            .build();
    return hFileContext;
}
 
Example 18
Source Project: hbase   Source File: BoundedRecoveredHFilesOutputSink.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return Returns a base HFile without compressions or encodings; good enough for recovery
 *   given hfile has metadata on how it was written.
 */
private StoreFileWriter createRecoveredHFileWriter(TableName tableName, String regionName,
    long seqId, String familyName, boolean isMetaTable) throws IOException {
  Path outputDir = WALSplitUtil.tryCreateRecoveredHFilesDir(walSplitter.rootFS, walSplitter.conf,
    tableName, regionName, familyName);
  StoreFileWriter.Builder writerBuilder =
      new StoreFileWriter.Builder(walSplitter.conf, CacheConfig.DISABLED, walSplitter.rootFS)
          .withOutputDir(outputDir);
  HFileContext hFileContext = new HFileContextBuilder().
    withChecksumType(HStore.getChecksumType(walSplitter.conf)).
    withBytesPerCheckSum(HStore.getBytesPerChecksum(walSplitter.conf)).
    withCellComparator(isMetaTable?
      CellComparatorImpl.META_COMPARATOR: CellComparatorImpl.COMPARATOR).build();
  return writerBuilder.withFileContext(hFileContext).build();
}
 
Example 19
Source Project: hbase   Source File: CompressionTest.java    License: Apache License 2.0 5 votes vote down vote up
public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
  Configuration conf = HBaseConfiguration.create();
  HFileContext context = new HFileContextBuilder()
                         .withCompression(HFileWriterImpl.compressionByName(codec)).build();
  HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
      .withPath(fs, path)
      .withFileContext(context)
      .create();
  // Write any-old Cell...
  final byte [] rowKey = Bytes.toBytes("compressiontestkey");
  Cell c = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(rowKey)
    .setFamily(HConstants.EMPTY_BYTE_ARRAY)
    .setQualifier(HConstants.EMPTY_BYTE_ARRAY)
    .setTimestamp(HConstants.LATEST_TIMESTAMP)
    .setType(KeyValue.Type.Maximum.getCode())
    .setValue(Bytes.toBytes("compressiontestval"))
    .build();
  writer.append(c);
  writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval"));
  writer.close();
  Cell cc = null;
  HFile.Reader reader = HFile.createReader(fs, path, CacheConfig.DISABLED, true, conf);
  try {
    HFileScanner scanner = reader.getScanner(false, true);
    scanner.seekTo(); // position to the start of file
    // Scanner does not do Cells yet. Do below for now till fixed.
    cc = scanner.getCell();
    if (CellComparator.getInstance().compareRows(c, cc) != 0) {
      throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString());
    }
  } finally {
    reader.close();
  }
}
 
Example 20
Source Project: hbase   Source File: TestMobStoreCompaction.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create an HFile with the given number of bytes
 */
private void createHFile(Path path, int rowIdx, byte[] dummyData) throws IOException {
  HFileContext meta = new HFileContextBuilder().build();
  HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
      .withFileContext(meta).create();
  long now = System.currentTimeMillis();
  try {
    KeyValue kv = new KeyValue(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)), COLUMN_FAMILY,
        Bytes.toBytes("colX"), now, dummyData);
    writer.append(kv);
  } finally {
    writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
    writer.close();
  }
}
 
Example 21
Source Project: hbase   Source File: HFileTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create an HFile with the given number of rows between a given
 * start key and end key @ family:qualifier.
 * If withTag is true, we add the rowKey as the tag value for
 * tagtype MOB_TABLE_NAME_TAG_TYPE
 */
public static void createHFile(
    Configuration configuration,
    FileSystem fs, Path path, DataBlockEncoding encoding,
    byte[] family, byte[] qualifier,
    byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException {
  HFileContext meta = new HFileContextBuilder()
      .withIncludesTags(withTag)
      .withDataBlockEncoding(encoding)
      .withColumnFamily(family)
      .build();
  HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration))
      .withPath(fs, path)
      .withFileContext(meta)
      .create();
  long now = System.currentTimeMillis();
  try {
    // subtract 2 since iterateOnSplits doesn't include boundary keys
    for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) {
      Cell kv = new KeyValue(key, family, qualifier, now, key);
      if (withTag) {
        // add a tag.  Arbitrarily chose mob tag since we have a helper already.
        Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, key);
        kv = MobUtils.createMobRefCell(kv, key, tableNameTag);

        // verify that the kv has the tag.
        Optional<Tag> tag = PrivateCellUtil.getTag(kv, TagType.MOB_TABLE_NAME_TAG_TYPE);
        if (!tag.isPresent()) {
          throw new IllegalStateException("Tag didn't stick to KV " + kv.toString());
        }
      }
      writer.append(kv);
    }
  } finally {
    writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
    writer.close();
  }
}
 
Example 22
@Test
public void testReseek() throws Exception {
  // write the file
  Path f = new Path(ROOT_DIR, "testReseek");
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).withIncludesTags(true)
      .withCompressTags(true).withDataBlockEncoding(DataBlockEncoding.PREFIX).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(f)
      .withFileContext(meta).build();

  writeStoreFile(writer);
  writer.close();

  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
  HFileInfo fileInfo = new HFileInfo(context, conf);
  StoreFileReader reader =
      new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
  fileInfo.initMetaAndIndex(reader.getHFileReader());
  StoreFileScanner s = reader.getStoreFileScanner(false, false, false, 0, 0, false);
  try {
    // Now do reseek with empty KV to position to the beginning of the file
    KeyValue k = KeyValueUtil.createFirstOnRow(Bytes.toBytes("k2"));
    s.reseek(k);
    Cell kv = s.next();
    kv = s.next();
    kv = s.next();
    byte[] key5 = Bytes.toBytes("k5");
    assertTrue(Bytes.equals(key5, 0, key5.length, kv.getRowArray(), kv.getRowOffset(),
        kv.getRowLength()));
    List<Tag> tags = PrivateCellUtil.getTags(kv);
    assertEquals(1, tags.size());
    assertEquals("tag3", Bytes.toString(Tag.cloneValue(tags.get(0))));
  } finally {
    s.close();
  }
}
 
Example 23
Source Project: hbase   Source File: TestScannerWithBulkload.java    License: Apache License 2.0 5 votes vote down vote up
private Path writeToHFile(long l, String hFilePath, String pathStr, boolean nativeHFile)
    throws IOException {
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  final Path hfilePath = new Path(hFilePath);
  fs.mkdirs(hfilePath);
  Path path = new Path(pathStr);
  HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
  Assert.assertNotNull(wf);
  HFileContext context = new HFileContextBuilder().build();
  HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create();
  KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l,
      Bytes.toBytes("version2"));

  // Set cell seq id to test bulk load native hfiles.
  if (nativeHFile) {
    // Set a big seq id. Scan should not look at this seq id in a bulk loaded file.
    // Scan should only look at the seq id appended at the bulk load time, and not skip
    // this kv.
    kv.setSequenceId(9999999);
  }

  writer.append(kv);

  if (nativeHFile) {
    // Set a big MAX_SEQ_ID_KEY. Scan should not look at this seq id in a bulk loaded file.
    // Scan should only look at the seq id appended at the bulk load time, and not skip its
    // kv.
    writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(new Long(9999999)));
  }
  else {
  writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
  }
  writer.close();
  return hfilePath;
}
 
Example 24
Source Project: hbase   Source File: TestHStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test for hbase-1686.
 * @throws IOException
 */
@Test
public void testEmptyStoreFile() throws IOException {
  init(this.name.getMethodName());
  // Write a store file.
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null);
  flush(1);
  // Now put in place an empty store file.  Its a little tricky.  Have to
  // do manually with hacked in sequence id.
  HStoreFile f = this.store.getStorefiles().iterator().next();
  Path storedir = f.getPath().getParent();
  long seqid = f.getMaxSequenceId();
  Configuration c = HBaseConfiguration.create();
  FileSystem fs = FileSystem.get(c);
  HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
  StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c),
      fs)
          .withOutputDir(storedir)
          .withFileContext(meta)
          .build();
  w.appendMetadata(seqid + 1, false);
  w.close();
  this.store.close();
  // Reopen it... should pick up two files
  this.store =
      new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c, false);
  assertEquals(2, this.store.getStorefilesCount());

  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(),
      qualifiers);
  assertEquals(1, result.size());
}
 
Example 25
Source Project: hbase   Source File: TestHStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHFileContextSetWithCFAndTable() throws Exception {
  init(this.name.getMethodName());
  StoreFileWriter writer = store.createWriterInTmp(10000L,
      Compression.Algorithm.NONE, false, true, false, true);
  HFileContext hFileContext = writer.getHFileWriter().getFileContext();
  assertArrayEquals(family, hFileContext.getColumnFamily());
  assertArrayEquals(table, hFileContext.getTableName());
}
 
Example 26
Source Project: hbase   Source File: TestSecureBulkLoadManager.java    License: Apache License 2.0 5 votes vote down vote up
private void prepareHFile(Path dir, byte[] key, byte[] value) throws Exception {
  TableDescriptor desc = testUtil.getAdmin().getDescriptor(TABLE);
  ColumnFamilyDescriptor family = desc.getColumnFamily(FAMILY);
  Compression.Algorithm compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;

  CacheConfig writerCacheConf = new CacheConfig(conf, family, null, ByteBuffAllocator.HEAP);
  writerCacheConf.setCacheDataOnWrite(false);
  HFileContext hFileContext = new HFileContextBuilder()
      .withIncludesMvcc(false)
      .withIncludesTags(true)
      .withCompression(compression)
      .withCompressTags(family.isCompressTags())
      .withChecksumType(HStore.getChecksumType(conf))
      .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
      .withBlockSize(family.getBlocksize())
      .withHBaseCheckSum(true)
      .withDataBlockEncoding(family.getDataBlockEncoding())
      .withEncryptionContext(Encryption.Context.NONE)
      .withCreateTime(EnvironmentEdgeManager.currentTime())
      .build();
  StoreFileWriter.Builder builder =
      new StoreFileWriter.Builder(conf, writerCacheConf, dir.getFileSystem(conf))
      .withOutputDir(new Path(dir, family.getNameAsString()))
      .withBloomType(family.getBloomFilterType())
      .withMaxKeyCount(Integer.MAX_VALUE)
      .withFileContext(hFileContext);
  StoreFileWriter writer = builder.build();

  Put put = new Put(key);
  put.addColumn(FAMILY, COLUMN, value);
  for (Cell c : put.get(FAMILY, COLUMN)) {
    writer.append(c);
  }

  writer.close();
}
 
Example 27
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStoreFileReference() throws Exception {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testStoreFileReference")).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
    new Path(testDir, hri.getTable().getNameAsString()), hri);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();

  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);
  Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  writer.close();

  HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);
  file.initReader();
  StoreFileReader r = file.getReader();
  assertNotNull(r);
  StoreFileScanner scanner =
    new StoreFileScanner(r, mock(HFileScanner.class), false, false, 0, 0, false);

  // Verify after instantiating scanner refCount is increased
  assertTrue("Verify file is being referenced", file.isReferencedInReads());
  scanner.close();
  // Verify after closing scanner refCount is decreased
  assertFalse("Verify file is not being referenced", file.isReferencedInReads());
}
 
Example 28
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHFileLink() throws IOException {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build();
  // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
  Configuration testConf = new Configuration(this.conf);
  CommonFSUtils.setRootDir(testConf, testDir);
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();

  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);

  Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY));
  HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
  Path linkFilePath =
    new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));

  // Try to open store file from link
  StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true);
  HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);
  assertTrue(storeFileInfo.isLink());
  hsf.initReader();

  // Now confirm that I can read from the link
  int count = 1;
  HFileScanner s = hsf.getReader().getScanner(false, false);
  s.seekTo();
  while (s.next()) {
    count++;
  }
  assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}
 
Example 29
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBloomFilter() throws Exception {
  conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
  conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);

  // write the file
  Path f = new Path(ROOT_DIR, name.getMethodName());
  HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)
    .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
    .withBloomType(BloomType.ROW).withMaxKeyCount(2000).withFileContext(meta).build();
  bloomWriteRead(writer, fs);
}
 
Example 30
Source Project: hbase   Source File: TestHStoreFile.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test for HBASE-8012
 */
@Test
public void testReseek() throws Exception {
  // write the file
  Path f = new Path(ROOT_DIR, name.getMethodName());
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)
    .withFileContext(meta).build();

  writeStoreFile(writer);
  writer.close();

  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
  HFileInfo fileInfo = new HFileInfo(context, conf);
  StoreFileReader reader =
    new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
  fileInfo.initMetaAndIndex(reader.getHFileReader());

  // Now do reseek with empty KV to position to the beginning of the file

  KeyValue k = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY);
  StoreFileScanner s = getStoreFileScanner(reader, false, false);
  s.reseek(k);

  assertNotNull("Intial reseek should position at the beginning of the file", s.peek());
}