Java Code Examples for org.apache.hadoop.hbase.fs.HFileSystem#get()

@Before
public void setUp() throws IOException {
  conf = TEST_UTIL.getConfiguration();
  this.conf.set("dfs.datanode.data.dir.perm", "700");
  conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_BLOCK_SIZE);
  conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE);
  conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, cacheCompressedData);
  cowType.modifyConf(conf);
  conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.DATA));
  conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
      cowType.shouldBeCached(BlockType.LEAF_INDEX));
  conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
      cowType.shouldBeCached(BlockType.BLOOM_CHUNK));
  cacheConf = new CacheConfig(conf, blockCache);
  fs = HFileSystem.get(conf);
}
 

@Override public int run(String[] args) throws Exception {
  if (args.length < 1) {
    System.err.println("Usage: Clean <output dir>");
    return -1;
  }

  Path p = new Path(args[0]);
  Configuration conf = getConf();
  TableName tableName = getTableName(conf);
  try (FileSystem fs = HFileSystem.get(conf);
      Connection conn = ConnectionFactory.createConnection(conf);
      Admin admin = conn.getAdmin()) {
    if (admin.tableExists(tableName)) {
      admin.disableTable(tableName);
      admin.deleteTable(tableName);
    }

    if (fs.exists(p)) {
      fs.delete(p, true);
    }
  }

  return 0;
}
 

@Before
public void setUp() throws IOException {
  // parameterized tests add [#] suffix get rid of [ and ].
  table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_"));

  conf = TEST_UTIL.getConfiguration();
  conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
  conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
  conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false);
  conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false);
  fs = HFileSystem.get(conf);

  // Create the schema
  ColumnFamilyDescriptor hcd = cowType
      .modifyFamilySchema(
        ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL))
      .build();
  TableDescriptor htd =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(hcd).build();

  // Create a store based on the schema
  String id = TestCacheOnWriteInSchema.class.getName();
  Path logdir =
    new Path(CommonFSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id));
  fs.delete(logdir, true);

  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  walFactory = new WALFactory(conf, id);

  region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info));
  region.setBlockCache(BlockCacheFactory.createBlockCache(conf));
  store = new HStore(region, hcd, conf, false);
}
 

@Before
public void setUp() throws IOException {
  keys.clear();
  rand = new Random(2389757);
  firstKeyInFile = null;
  conf = TEST_UTIL.getConfiguration();

  // This test requires at least HFile format version 2.
  conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);

  fs = HFileSystem.get(conf);
}
 

private void clear() throws IOException {
  keys.clear();
  rand = new Random(2389757);
  firstKeyInFile = null;
  conf = TEST_UTIL.getConfiguration();

  // This test requires at least HFile format version 2.
  conf.setInt(HFile.FORMAT_VERSION_KEY, 3);

  fs = HFileSystem.get(conf);
}
 

@Before
public void setUp() throws IOException {
  conf = TEST_UTIL.getConfiguration();
  conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
  fs = HFileSystem.get(conf);
  blockCache = BlockCacheFactory.createBlockCache(conf);
  cacheConf = new CacheConfig(conf, blockCache);
}
 

public FileSystem getTestFileSystem() throws IOException {
  return HFileSystem.get(conf);
}
 

/**
 * Scanner.seekBefore() could fail because when seeking to a previous HFile data block, it needs
 * to know the size of that data block, which it calculates using current data block offset and
 * the previous data block offset.  This fails to work when there are leaf-level index blocks in
 * the scannable section of the HFile, i.e. starting in HFileV2.  This test will try seekBefore()
 * on a flat (single-level) and multi-level (2,3) HFile and confirm this bug is now fixed.  This
 * bug also happens for inline Bloom blocks for the same reasons.
 */
@Test
public void testMultiIndexLevelRandomHFileWithBlooms() throws IOException {
  conf = TEST_UTIL.getConfiguration();
  TEST_UTIL.getConfiguration().setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);

  // Try out different HFile versions to ensure reverse scan works on each version
  for (int hfileVersion = HFile.MIN_FORMAT_VERSION_WITH_TAGS;
          hfileVersion <= HFile.MAX_FORMAT_VERSION; hfileVersion++) {

    conf.setInt(HFile.FORMAT_VERSION_KEY, hfileVersion);
    fs = HFileSystem.get(conf);

    // Try out different bloom types because inline Bloom blocks break seekBefore()
    for (BloomType bloomType : BloomType.values()) {

      // Test out HFile block indices of various sizes/levels
      for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; testI++) {
        int indexBlockSize = INDEX_CHUNK_SIZES[testI];
        int expectedNumLevels = EXPECTED_NUM_LEVELS[testI];

        LOG.info(String.format("Testing HFileVersion: %s, BloomType: %s, Index Levels: %s",
          hfileVersion, bloomType, expectedNumLevels));

        conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize);
        conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE);
        conf.setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);

        Cell[] cells = new Cell[NUM_KV];

        Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
          String.format("testMultiIndexLevelRandomHFileWithBlooms-%s-%s-%s",
            hfileVersion, bloomType, testI));

        // Disable caching to prevent it from hiding any bugs in block seeks/reads
        conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
        CacheConfig cacheConf = new CacheConfig(conf);

        // Write the HFile
        {
          HFileContext meta = new HFileContextBuilder()
                              .withBlockSize(DATA_BLOCK_SIZE)
                              .build();

          StoreFileWriter storeFileWriter =
              new StoreFileWriter.Builder(conf, cacheConf, fs)
            .withFilePath(hfilePath)
            .withFileContext(meta)
            .withBloomType(bloomType)
            .build();

          for (int i = 0; i < NUM_KV; i++) {
            byte[] row = RandomKeyValueUtil.randomOrderedKey(RAND, i);
            byte[] qual = RandomKeyValueUtil.randomRowOrQualifier(RAND);
            byte[] value = RandomKeyValueUtil.randomValue(RAND);
            KeyValue kv = new KeyValue(row, FAM, qual, value);

            storeFileWriter.append(kv);
            cells[i] = kv;
          }

          storeFileWriter.close();
        }

        // Read the HFile
        HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, true, conf);

        // Sanity check the HFile index level
        assertEquals(expectedNumLevels, reader.getTrailer().getNumDataIndexLevels());

        // Check that we can seekBefore in either direction and with both pread
        // enabled and disabled
        for (boolean pread : new boolean[] { false, true }) {
          HFileScanner scanner = reader.getScanner(true, pread);
          checkNoSeekBefore(cells, scanner, 0);
          for (int i = 1; i < NUM_KV; i++) {
            checkSeekBefore(cells, scanner, i);
            checkCell(cells[i-1], scanner.getCell());
          }
          assertTrue(scanner.seekTo());
          for (int i = NUM_KV - 1; i >= 1; i--) {
            checkSeekBefore(cells, scanner, i);
            checkCell(cells[i-1], scanner.getCell());
          }
          checkNoSeekBefore(cells, scanner, 0);
          scanner.close();
        }

        reader.close();
      }
    }
  }
}
 

@Before
public void setUp() throws IOException {
  fs = HFileSystem.get(TEST_UTIL.getConfiguration());
}
 

@Before
public void setUp() throws Exception {
  fs = HFileSystem.get(TEST_UTIL.getConfiguration());
  hfs = (HFileSystem)fs;
}