Java Code Examples for org.apache.hadoop.hbase.io.hfile.HFileScanner#next()

/**
 *
 * @param s
 * @param k
 * @return false if not found or if k is after the end.
 * @throws IOException
 */
public static boolean seekAtOrAfter(HFileScanner s, Cell k)
throws IOException {
  int result = s.seekTo(k);
  if(result < 0) {
    if (result == HConstants.INDEX_KEY_MAGIC) {
      // using faked key
      return true;
    }
    // Passed KV is smaller than first KV in file, work from start of file
    return s.seekTo();
  } else if(result > 0) {
    // Passed KV is larger than current KV in file, if there is a next
    // it is the "after", if not then this scanner is done.
    return s.next();
  }
  // Seeked to the exact key
  return true;
}
 

static boolean reseekAtOrAfter(HFileScanner s, Cell k)
throws IOException {
  //This function is similar to seekAtOrAfter function
  int result = s.reseekTo(k);
  if (result <= 0) {
    if (result == HConstants.INDEX_KEY_MAGIC) {
      // using faked key
      return true;
    }
    // If up to now scanner is not seeked yet, this means passed KV is smaller
    // than first KV in file, and it is the first time we seek on this file.
    // So we also need to work from the start of file.
    if (!s.isSeeked()) {
      return  s.seekTo();
    }
    return true;
  }
  // passed KV is larger than current KV in file, if there is a next
  // it is after, if not then this scanner is done.
  return s.next();
}
 

private void verifyCounts(int countRow1, int countRow2) throws Exception {
  int count1 = 0;
  int count2 = 0;
  for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
    HFileScanner scanner = f.getReader().getScanner(false, false);
    scanner.seekTo();
    do {
      byte[] row = CellUtil.cloneRow(scanner.getCell());
      if (Bytes.equals(row, STARTROW)) {
        count1++;
      } else if (Bytes.equals(row, secondRowBytes)) {
        count2++;
      }
    } while (scanner.next());
  }
  assertEquals(countRow1, count1);
  assertEquals(countRow2, count2);
}
 

@Override
void doRow(int i) throws Exception {
  HFileScanner scanner = this.reader.getScanner(false, false);
  byte [] b = getRandomRow();
  // System.out.println("Random row: " + new String(b));
  Cell c = createCell(b);
  if (scanner.seekTo(c) != 0) {
    LOG.info("Nonexistent row: " + new String(b));
    return;
  }
  // TODO: HFileScanner doesn't do Cells yet. Temporary fix.
  c = scanner.getCell();
  // System.out.println("Found row: " +
  //  new String(c.getRowArray(), c.getRowOffset(), c.getRowLength()));
  PerformanceEvaluationCommons.assertKey(b, c);
  for (int ii = 0; ii < 30; ii++) {
    if (!scanner.next()) {
      LOG.info("NOTHING FOLLOWS");
      return;
    }
    c = scanner.getCell();
    PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH);
  }
}
 

private int readAndCheckbytes(HFileScanner scanner, int start, int n)
    throws IOException {
  String value = "value";
  int i = start;
  for (; i < (start + n); i++) {
    ByteBuffer key = scanner.getKey();
    ByteBuffer val = scanner.getValue();
    String keyStr = String.format(LOCAL_FORMATTER, Integer.valueOf(i));
    String valStr = value + keyStr;
    KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
        Bytes.toBytes("qual"), Bytes.toBytes(valStr));
    byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0,
        Bytes.toBytes(key).length).getKey();
    assertArrayEquals(kv.getKey(), keyBytes,
        "bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)));
    byte[] valBytes = Bytes.toBytes(val);
    assertArrayEquals(Bytes.toBytes(valStr), valBytes,
        "bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes));
    if (!scanner.next()) {
      break;
    }
  }
  assertEquals(i, start + n - 1);
  return (start + n);
}
 

@Override
void doRow(int i) throws Exception {
  HFileScanner scanner = this.reader.getScanner(false, true);
  byte[] gaussianRandomRowBytes = getGaussianRandomRowBytes();
  scanner.seekTo(createCell(gaussianRandomRowBytes));
  for (int ii = 0; ii < 30; ii++) {
    if (!scanner.next()) {
      LOG.info("NOTHING FOLLOWS");
      return;
    }
    // TODO: Fix. Make scanner do Cells.
    scanner.getCell();
  }
}
 

/**
 * Method returns the total KVs in given hfile
 * @param fs File System
 * @param p HFile path
 * @return KV count in the given hfile
 * @throws IOException
 */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf);
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  reader.close();
  return count;
}
 

/**
 * Method returns the total KVs in given hfile
 * @param fs File System
 * @param p HFile path
 * @return KV count in the given hfile
 * @throws IOException
 */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf);
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  reader.close();
  return count;
}
 

@Test
public void testHFileLink() throws IOException {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build();
  // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
  Configuration testConf = new Configuration(this.conf);
  CommonFSUtils.setRootDir(testConf, testDir);
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();

  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);

  Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY));
  HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
  Path linkFilePath =
    new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));

  // Try to open store file from link
  StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true);
  HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);
  assertTrue(storeFileInfo.isLink());
  hsf.initReader();

  // Now confirm that I can read from the link
  int count = 1;
  HFileScanner s = hsf.getReader().getScanner(false, false);
  s.seekTo();
  while (s.next()) {
    count++;
  }
  assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}
 

private int verifyHFile(Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader =
    HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf);
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  assertTrue(count > 0);
  reader.close();
  return count;
}
 

/**
 * Test that our mechanism of writing store files in one region to reference store files in other
 * regions works.
 */
@Test
public void testReference() throws IOException {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testReferenceTb")).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
    new Path(testDir, hri.getTable().getNameAsString()), hri);

  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);

  Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);
  hsf.initReader();
  StoreFileReader reader = hsf.getReader();
  // Split on a row, not in middle of row. Midkey returned by reader
  // may be in middle of row. Create new one with empty column and
  // timestamp.
  byte[] midRow = CellUtil.cloneRow(reader.midKey().get());
  byte[] finalRow = CellUtil.cloneRow(reader.getLastKey().get());
  hsf.closeStoreFile(true);

  // Make a reference
  RegionInfo splitHri = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(midRow).build();
  Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true);
  HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true);
  refHsf.initReader();
  // Now confirm that I can read from the reference and that it only gets
  // keys from top half of the file.
  HFileScanner s = refHsf.getReader().getScanner(false, false);
  Cell kv = null;
  for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
    ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey());
    kv = KeyValueUtil.createKeyValueFromKey(bb);
    if (first) {
      assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0,
        midRow.length));
      first = false;
    }
  }
  assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0,
    finalRow.length));
}
 

/**
 * This throws a WrongRegionException if the HFile does not fit in this region, or an
 * InvalidHFileException if the HFile is not valid.
 */
public void assertBulkLoadHFileOk(Path srcPath) throws IOException {
  HFile.Reader reader  = null;
  try {
    LOG.info("Validating hfile at " + srcPath + " for inclusion in " + this);
    FileSystem srcFs = srcPath.getFileSystem(conf);
    srcFs.access(srcPath, FsAction.READ_WRITE);
    reader = HFile.createReader(srcFs, srcPath, cacheConf, isPrimaryReplicaStore(), conf);

    Optional<byte[]> firstKey = reader.getFirstRowKey();
    Preconditions.checkState(firstKey.isPresent(), "First key can not be null");
    Optional<Cell> lk = reader.getLastKey();
    Preconditions.checkState(lk.isPresent(), "Last key can not be null");
    byte[] lastKey =  CellUtil.cloneRow(lk.get());

    if (LOG.isDebugEnabled()) {
      LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey.get()) +
          " last=" + Bytes.toStringBinary(lastKey));
      LOG.debug("Region bounds: first=" +
          Bytes.toStringBinary(getRegionInfo().getStartKey()) +
          " last=" + Bytes.toStringBinary(getRegionInfo().getEndKey()));
    }

    if (!this.getRegionInfo().containsRange(firstKey.get(), lastKey)) {
      throw new WrongRegionException(
          "Bulk load file " + srcPath.toString() + " does not fit inside region "
          + this.getRegionInfo().getRegionNameAsString());
    }

    if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE)) {
      LOG.warn("Trying to bulk load hfile " + srcPath + " with size: " +
          reader.length() + " bytes can be problematic as it may lead to oversplitting.");
    }

    if (verifyBulkLoads) {
      long verificationStartTime = EnvironmentEdgeManager.currentTime();
      LOG.info("Full verification started for bulk load hfile: {}", srcPath);
      Cell prevCell = null;
      HFileScanner scanner = reader.getScanner(false, false, false);
      scanner.seekTo();
      do {
        Cell cell = scanner.getCell();
        if (prevCell != null) {
          if (comparator.compareRows(prevCell, cell) > 0) {
            throw new InvalidHFileException("Previous row is greater than"
                + " current row: path=" + srcPath + " previous="
                + CellUtil.getCellKeyAsString(prevCell) + " current="
                + CellUtil.getCellKeyAsString(cell));
          }
          if (CellComparator.getInstance().compareFamilies(prevCell, cell) != 0) {
            throw new InvalidHFileException("Previous key had different"
                + " family compared to current key: path=" + srcPath
                + " previous="
                + Bytes.toStringBinary(prevCell.getFamilyArray(), prevCell.getFamilyOffset(),
                    prevCell.getFamilyLength())
                + " current="
                + Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
                    cell.getFamilyLength()));
          }
        }
        prevCell = cell;
      } while (scanner.next());
      LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() +
        " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) + " ms");
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

/**
 * This test creates an hfile and then the dir structures and files to verify that references to
 * hfilelinks (created by snapshot clones) can be properly interpreted.
 */
@Test
public void testReferenceToHFileLink() throws IOException {
  // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
  Configuration testConf = new Configuration(this.conf);
  CommonFSUtils.setRootDir(testConf, testDir);

  // adding legal table name chars to verify regex handles it.
  RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("_original-evil-name")).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);

  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
  StoreFileWriter writer = new StoreFileWriter.Builder(testConf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);
  Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());

  // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
  RegionInfo hriClone = RegionInfoBuilder.newBuilder(TableName.valueOf("clone")).build();
  HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hriClone);
  Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);
  HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
  Path linkFilePath =
    new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));

  // create splits of the link.
  // <root>/clone/splitA/<cf>/<reftohfilelink>,
  // <root>/clone/splitB/<cf>/<reftohfilelink>
  RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build();
  RegionInfo splitHriB =
    RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build();
  HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true);
  f.initReader();
  Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top
  Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom
  f.closeStoreFile(true);
  // OK test the thing
  CommonFSUtils.logFileSystemState(fs, testDir, LOG);

  // There is a case where a file with the hfilelink pattern is actually a daughter
  // reference to a hfile link. This code in StoreFile that handles this case.

  // Try to open store file from link
  HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true);
  hsfA.initReader();

  // Now confirm that I can read from the ref to link
  int count = 1;
  HFileScanner s = hsfA.getReader().getScanner(false, false);
  s.seekTo();
  while (s.next()) {
    count++;
  }
  assertTrue(count > 0); // read some rows here

  // Try to open store file from link
  HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true);
  hsfB.initReader();

  // Now confirm that I can read from the ref to link
  HFileScanner sB = hsfB.getReader().getScanner(false, false);
  sB.seekTo();

  // count++ as seekTo() will advance the scanner
  count++;
  while (sB.next()) {
    count++;
  }

  // read the rest of the rows
  assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}
 

/**
 * Injects errors into the pread calls of an on-disk file, and makes
 * sure those bubble up to the HFile scanner
 */
@Test
public void testHFileScannerThrowsErrors() throws IOException {
  Path hfilePath = new Path(new Path(
      util.getDataTestDir("internalScannerExposesErrors"),
      "regionname"), "familyname");
  HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
  FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
  FileSystem fs = new HFileSystem(faultyfs);
  CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
  HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
  StoreFileWriter writer = new StoreFileWriter.Builder(
      util.getConfiguration(), cacheConf, hfs)
          .withOutputDir(hfilePath)
          .withFileContext(meta)
          .build();
  TestHStoreFile.writeStoreFile(
      writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));

  HStoreFile sf = new HStoreFile(fs, writer.getPath(), util.getConfiguration(), cacheConf,
      BloomType.NONE, true);
  sf.initReader();
  StoreFileReader reader = sf.getReader();
  HFileScanner scanner = reader.getScanner(false, true);

  FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
  assertNotNull(inStream);

  scanner.seekTo();
  // Do at least one successful read
  assertTrue(scanner.next());

  faultyfs.startFaults();

  try {
    int scanned=0;
    while (scanner.next()) {
      scanned++;
    }
    fail("Scanner didn't throw after faults injected");
  } catch (IOException ioe) {
    LOG.info("Got expected exception", ioe);
    assertTrue(ioe.getMessage().contains("Fault"));
  }
  reader.close(true); // end of test so evictOnClose
}
 

@Test
public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
  // disable the store file refresh chore (we do this by hand)
  HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
  restartRegionServer();

  try {
    LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
    openRegion(HTU, getRS(), hriSecondary);

    // load some data to primary
    LOG.info("Loading data to primary region");
    for (int i = 0; i < 3; ++i) {
      HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
      HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);
    }

    HRegion primaryRegion = getRS().getRegion(hriPrimary.getEncodedName());
    Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());

    // Refresh store files on the secondary
    Region secondaryRegion = getRS().getRegion(hriSecondary.getEncodedName());
    secondaryRegion.getStore(f).refreshStoreFiles();
    Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());

    // force compaction
    LOG.info("Force Major compaction on primary region " + hriPrimary);
    primaryRegion.compact(true);
    Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
    List<RegionServerThread> regionServerThreads = HTU.getMiniHBaseCluster()
        .getRegionServerThreads();
    HRegionServer hrs = null;
    for (RegionServerThread rs : regionServerThreads) {
      if (rs.getRegionServer()
          .getOnlineRegion(primaryRegion.getRegionInfo().getRegionName()) != null) {
        hrs = rs.getRegionServer();
        break;
      }
    }
    CompactedHFilesDischarger cleaner =
        new CompactedHFilesDischarger(100, null, hrs, false);
    cleaner.chore();
    // scan all the hfiles on the secondary.
    // since there are no read on the secondary when we ask locations to
    // the NN a FileNotFound exception will be returned and the FileLink
    // should be able to deal with it giving us all the result we expect.
    int keys = 0;
    int sum = 0;
    for (HStoreFile sf : ((HStore) secondaryRegion.getStore(f)).getStorefiles()) {
      // Our file does not exist anymore. was moved by the compaction above.
      LOG.debug(Boolean.toString(getRS().getFileSystem().exists(sf.getPath())));
      Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));

      HFileScanner scanner = sf.getReader().getScanner(false, false);
      scanner.seekTo();
      do {
        keys++;

        Cell cell = scanner.getCell();
        sum += Integer.parseInt(Bytes.toString(cell.getRowArray(),
          cell.getRowOffset(), cell.getRowLength()));
      } while (scanner.next());
    }
    Assert.assertEquals(3000, keys);
    Assert.assertEquals(4498500, sum);
  } finally {
    HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
    closeRegion(HTU, getRS(), hriSecondary);
  }
}
 

/**
 * Copy half of an HFile into a new HFile.
 */
private static void copyHFileHalf(Configuration conf, Path inFile, Path outFile,
    Reference reference, ColumnFamilyDescriptor familyDescriptor) throws IOException {
  FileSystem fs = inFile.getFileSystem(conf);
  CacheConfig cacheConf = CacheConfig.DISABLED;
  HalfStoreFileReader halfReader = null;
  StoreFileWriter halfWriter = null;
  try {
    ReaderContext context = new ReaderContextBuilder()
        .withFileSystemAndPath(fs, inFile).build();
    HFileInfo hfile = new HFileInfo(context, conf);
    halfReader = new HalfStoreFileReader(context, hfile, cacheConf, reference,
      new AtomicInteger(0), conf);
    hfile.initMetaAndIndex(halfReader.getHFileReader());
    Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();

    int blocksize = familyDescriptor.getBlocksize();
    Algorithm compression = familyDescriptor.getCompressionType();
    BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
    HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
      .withChecksumType(HStore.getChecksumType(conf))
      .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blocksize)
      .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
      .build();
    halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
      .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
    HFileScanner scanner = halfReader.getScanner(false, false, false);
    scanner.seekTo();
    do {
      halfWriter.append(scanner.getCell());
    } while (scanner.next());

    for (Map.Entry<byte[], byte[]> entry : fileInfo.entrySet()) {
      if (shouldCopyHFileMetaKey(entry.getKey())) {
        halfWriter.appendFileInfo(entry.getKey(), entry.getValue());
      }
    }
  } finally {
    if (halfReader != null) {
      try {
        halfReader.close(cacheConf.shouldEvictOnClose());
      } catch (IOException e) {
        LOG.warn("failed to close hfile reader for " + inFile, e);
      }
    }
    if (halfWriter != null) {
      halfWriter.close();
    }
  }
}
 

@Test
public void testWrite() throws Exception {
  Configuration conf = new Configuration();
  HColumnDescriptor columnDescriptor = new HColumnDescriptor();
  // Disable block cache to ensure it reads the actual file content.
  columnDescriptor.setBlockCacheEnabled(false);
  FileSystem fs = FileSystem.get(conf);
  int blockSize = conf.getInt(Constants.HFILE_BLOCKSIZE, 16384);
  final StoreFile.Writer writer =
      new StoreFile.WriterBuilder(conf, new CacheConfig(conf, columnDescriptor), fs, blockSize)
          .withFilePath(new Path(tempFile.toURI()))
          .build();
  /* Create our RecordWriter */
  RecordWriter<BytesWritable, BytesWritable> hfileWriter =
      new HFileRecordWriter(writer);

  List<String> keys = Lists.newArrayList();
  List<String> values = Lists.newArrayList();
  for (int i = 0; i < 100; ++i) {
    String key = String.format("%03d", i);
    String val = "value " + i;
    keys.add(key);
    values.add(val);
    hfileWriter.write(new BytesWritable(key.getBytes()), new BytesWritable(val.getBytes()));
  }
  /* This internally closes the StoreFile.Writer */
  hfileWriter.close(null);

  HFile.Reader reader = HFile.createReader(fs, new Path(tempFile.toURI()),
      new CacheConfig(conf, columnDescriptor));
  HFileScanner scanner = reader.getScanner(false, false, false);
  boolean valid = scanner.seekTo();
  List<String> gotKeys = Lists.newArrayListWithCapacity(keys.size());
  List<String> gotValues = Lists.newArrayListWithCapacity(values.size());
  while(valid) {
    KeyValue keyValue = scanner.getKeyValue();
    gotKeys.add(new String(keyValue.getRow()));
    gotValues.add(new String(keyValue.getValue()));
    valid = scanner.next();
  }
  assertEquals(keys, gotKeys);
  assertEquals(values, gotValues);
  reader.close();
}