Java Code Examples for org.apache.hadoop.hbase.io.hfile.HFileScanner#seekTo()

/**
 * Issues an HFile lookup on the underlying HFile.Reader. This is protected
 * for testing.
 */
protected Pair<ByteBuffer, Pair<ByteBuffer, Throwable>> getValueFromHFile(ByteBuffer key) {
  try {
    HFileScanner scanner = reader.getScanner(true, true, false);
    KeyValue kv = buildKeyValueForLookup(
        BytesUtil.readBytesFromByteBufferWithoutConsume(key));
    int code = scanner.seekTo(kv.getKey());
    ByteBuffer value = null;
    if (code == 0) {
      value = ByteBuffer.wrap(scanner.getKeyValue().getValue());
      if (this.sizeStatsKey != null) {
        Stats.addMetric(this.sizeStatsKey, value.remaining());
      }
      Stats.addMetric("value-size", value.remaining());
    } else {
      Stats.incr("not-found");
      if (this.notFoundStatsKey != null) {
        Stats.incr(this.notFoundStatsKey);
      }
    }
    return new ImmutablePair(key, new ImmutablePair(value, null));
  } catch (Throwable t) {
    return new ImmutablePair(key, new ImmutablePair(null, t));
  }
}
 

private void verifyCounts(int countRow1, int countRow2) throws Exception {
  int count1 = 0;
  int count2 = 0;
  for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
    HFileScanner scanner = f.getReader().getScanner(false, false);
    scanner.seekTo();
    do {
      byte[] row = CellUtil.cloneRow(scanner.getCell());
      if (Bytes.equals(row, STARTROW)) {
        count1++;
      } else if (Bytes.equals(row, secondRowBytes)) {
        count2++;
      }
    } while (scanner.next());
  }
  assertEquals(countRow1, count1);
  assertEquals(countRow2, count2);
}
 

/**
 *
 * @param s
 * @param k
 * @return false if not found or if k is after the end.
 * @throws IOException
 */
public static boolean seekAtOrAfter(HFileScanner s, Cell k)
throws IOException {
  int result = s.seekTo(k);
  if(result < 0) {
    if (result == HConstants.INDEX_KEY_MAGIC) {
      // using faked key
      return true;
    }
    // Passed KV is smaller than first KV in file, work from start of file
    return s.seekTo();
  } else if(result > 0) {
    // Passed KV is larger than current KV in file, if there is a next
    // it is the "after", if not then this scanner is done.
    return s.next();
  }
  // Seeked to the exact key
  return true;
}
 

static boolean reseekAtOrAfter(HFileScanner s, Cell k)
throws IOException {
  //This function is similar to seekAtOrAfter function
  int result = s.reseekTo(k);
  if (result <= 0) {
    if (result == HConstants.INDEX_KEY_MAGIC) {
      // using faked key
      return true;
    }
    // If up to now scanner is not seeked yet, this means passed KV is smaller
    // than first KV in file, and it is the first time we seek on this file.
    // So we also need to work from the start of file.
    if (!s.isSeeked()) {
      return  s.seekTo();
    }
    return true;
  }
  // passed KV is larger than current KV in file, if there is a next
  // it is after, if not then this scanner is done.
  return s.next();
}
 

@Override
void doRow(int i) throws Exception {
  HFileScanner scanner = this.reader.getScanner(false, false);
  byte [] b = getRandomRow();
  // System.out.println("Random row: " + new String(b));
  Cell c = createCell(b);
  if (scanner.seekTo(c) != 0) {
    LOG.info("Nonexistent row: " + new String(b));
    return;
  }
  // TODO: HFileScanner doesn't do Cells yet. Temporary fix.
  c = scanner.getCell();
  // System.out.println("Found row: " +
  //  new String(c.getRowArray(), c.getRowOffset(), c.getRowLength()));
  PerformanceEvaluationCommons.assertKey(b, c);
  for (int ii = 0; ii < 30; ii++) {
    if (!scanner.next()) {
      LOG.info("NOTHING FOLLOWS");
      return;
    }
    c = scanner.getCell();
    PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH);
  }
}
 

@Override
public Optional<Cell> getFirstKey() {
  if (!firstKeySeeked) {
    HFileScanner scanner = getScanner(true, true, false);
    try {
      if (scanner.seekTo()) {
        this.firstKey = Optional.ofNullable(scanner.getKey());
      }
      firstKeySeeked = true;
    } catch (IOException e) {
      LOG.warn("Failed seekTo first KV in the file", e);
    } finally {
      if(scanner != null) {
        scanner.close();
      }
    }
  }
  return this.firstKey;
}
 

/**
 * Creates an inner HFileScanner object for a given HFile path
 */
public static HFileScanner createScanner(FileSystem fs, Path path) throws IOException {
    Configuration config = fs.getConf();
    HFile.Reader reader = HFile.createReader(fs, path, getCacheConfig(config), config);

    HFileScanner scanner = reader.getScanner(false, false);
    scanner.seekTo();

    return scanner;
}
 

@Test
public void testHFileLink() throws IOException {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build();
  // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
  Configuration testConf = new Configuration(this.conf);
  CommonFSUtils.setRootDir(testConf, testDir);
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();

  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);

  Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY));
  HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
  Path linkFilePath =
    new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));

  // Try to open store file from link
  StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true);
  HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);
  assertTrue(storeFileInfo.isLink());
  hsf.initReader();

  // Now confirm that I can read from the link
  int count = 1;
  HFileScanner s = hsf.getReader().getScanner(false, false);
  s.seekTo();
  while (s.next()) {
    count++;
  }
  assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}
 

@Override
void doRow(int i) throws Exception {
  HFileScanner scanner = this.reader.getScanner(false, true);
  byte[] gaussianRandomRowBytes = getGaussianRandomRowBytes();
  scanner.seekTo(createCell(gaussianRandomRowBytes));
  for (int ii = 0; ii < 30; ii++) {
    if (!scanner.next()) {
      LOG.info("NOTHING FOLLOWS");
      return;
    }
    // TODO: Fix. Make scanner do Cells.
    scanner.getCell();
  }
}
 

@Override
void doRow(int i) throws Exception {
  HFileScanner scanner = this.reader.getScanner(false, true);
  byte [] b = getRandomRow();
  if (scanner.seekTo(createCell(b)) < 0) {
    LOG.info("Not able to seekTo " + new String(b));
    return;
  }
  // TODO: Fix scanner so it does Cells
  Cell c = scanner.getCell();
  PerformanceEvaluationCommons.assertKey(b, c);
  PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH);
}
 

private int verifyHFile(Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader =
    HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf);
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  assertTrue(count > 0);
  reader.close();
  return count;
}
 

/**
 * Method returns the total KVs in given hfile
 * @param fs File System
 * @param p HFile path
 * @return KV count in the given hfile
 * @throws IOException
 */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf);
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  reader.close();
  return count;
}
 

public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
  Configuration conf = HBaseConfiguration.create();
  HFileContext context = new HFileContextBuilder()
                         .withCompression(HFileWriterImpl.compressionByName(codec)).build();
  HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
      .withPath(fs, path)
      .withFileContext(context)
      .create();
  // Write any-old Cell...
  final byte [] rowKey = Bytes.toBytes("compressiontestkey");
  Cell c = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(rowKey)
    .setFamily(HConstants.EMPTY_BYTE_ARRAY)
    .setQualifier(HConstants.EMPTY_BYTE_ARRAY)
    .setTimestamp(HConstants.LATEST_TIMESTAMP)
    .setType(KeyValue.Type.Maximum.getCode())
    .setValue(Bytes.toBytes("compressiontestval"))
    .build();
  writer.append(c);
  writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval"));
  writer.close();
  Cell cc = null;
  HFile.Reader reader = HFile.createReader(fs, path, CacheConfig.DISABLED, true, conf);
  try {
    HFileScanner scanner = reader.getScanner(false, true);
    scanner.seekTo(); // position to the start of file
    // Scanner does not do Cells yet. Do below for now till fixed.
    cc = scanner.getCell();
    if (CellComparator.getInstance().compareRows(c, cc) != 0) {
      throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString());
    }
  } finally {
    reader.close();
  }
}
 

/**
 * Method returns the total KVs in given hfile
 * @param fs File System
 * @param p HFile path
 * @return KV count in the given hfile
 * @throws IOException
 */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf);
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  reader.close();
  return count;
}
 

@Test
public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
  // disable the store file refresh chore (we do this by hand)
  HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
  restartRegionServer();

  try {
    LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
    openRegion(HTU, getRS(), hriSecondary);

    // load some data to primary
    LOG.info("Loading data to primary region");
    for (int i = 0; i < 3; ++i) {
      HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
      HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
      region.flush(true);
    }

    HRegion primaryRegion = getRS().getRegion(hriPrimary.getEncodedName());
    Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());

    // Refresh store files on the secondary
    Region secondaryRegion = getRS().getRegion(hriSecondary.getEncodedName());
    secondaryRegion.getStore(f).refreshStoreFiles();
    Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());

    // force compaction
    LOG.info("Force Major compaction on primary region " + hriPrimary);
    primaryRegion.compact(true);
    Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
    List<RegionServerThread> regionServerThreads = HTU.getMiniHBaseCluster()
        .getRegionServerThreads();
    HRegionServer hrs = null;
    for (RegionServerThread rs : regionServerThreads) {
      if (rs.getRegionServer()
          .getOnlineRegion(primaryRegion.getRegionInfo().getRegionName()) != null) {
        hrs = rs.getRegionServer();
        break;
      }
    }
    CompactedHFilesDischarger cleaner =
        new CompactedHFilesDischarger(100, null, hrs, false);
    cleaner.chore();
    // scan all the hfiles on the secondary.
    // since there are no read on the secondary when we ask locations to
    // the NN a FileNotFound exception will be returned and the FileLink
    // should be able to deal with it giving us all the result we expect.
    int keys = 0;
    int sum = 0;
    for (HStoreFile sf : ((HStore) secondaryRegion.getStore(f)).getStorefiles()) {
      // Our file does not exist anymore. was moved by the compaction above.
      LOG.debug(Boolean.toString(getRS().getFileSystem().exists(sf.getPath())));
      Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));

      HFileScanner scanner = sf.getReader().getScanner(false, false);
      scanner.seekTo();
      do {
        keys++;

        Cell cell = scanner.getCell();
        sum += Integer.parseInt(Bytes.toString(cell.getRowArray(),
          cell.getRowOffset(), cell.getRowLength()));
      } while (scanner.next());
    }
    Assert.assertEquals(3000, keys);
    Assert.assertEquals(4498500, sum);
  } finally {
    HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
    closeRegion(HTU, getRS(), hriSecondary);
  }
}
 

/**
 * Run small MR job.
 */
@Ignore("Goes zombie too frequently; needs work. See HBASE-14563") @Test
public void testWritingPEData() throws Exception {
  Configuration conf = util.getConfiguration();
  Path testDir = util.getDataTestDirOnTestFS("testWritingPEData");
  FileSystem fs = testDir.getFileSystem(conf);

  // Set down this value or we OOME in eclipse.
  conf.setInt("mapreduce.task.io.sort.mb", 20);
  // Write a few files.
  long hregionMaxFilesize = 10 * 1024;
  conf.setLong(HConstants.HREGION_MAX_FILESIZE, hregionMaxFilesize);

  Job job = new Job(conf, "testWritingPEData");
  setupRandomGeneratorMapper(job, false);
  // This partitioner doesn't work well for number keys but using it anyways
  // just to demonstrate how to configure it.
  byte[] startKey = new byte[RandomKVGeneratingMapper.KEYLEN_DEFAULT];
  byte[] endKey = new byte[RandomKVGeneratingMapper.KEYLEN_DEFAULT];

  Arrays.fill(startKey, (byte)0);
  Arrays.fill(endKey, (byte)0xff);

  job.setPartitionerClass(SimpleTotalOrderPartitioner.class);
  // Set start and end rows for partitioner.
  SimpleTotalOrderPartitioner.setStartKey(job.getConfiguration(), startKey);
  SimpleTotalOrderPartitioner.setEndKey(job.getConfiguration(), endKey);
  job.setReducerClass(CellSortReducer.class);
  job.setOutputFormatClass(HFileOutputFormat2.class);
  job.setNumReduceTasks(4);
  job.getConfiguration().setStrings("io.serializations", conf.get("io.serializations"),
      MutationSerialization.class.getName(), ResultSerialization.class.getName(),
      CellSerialization.class.getName());

  FileOutputFormat.setOutputPath(job, testDir);
  assertTrue(job.waitForCompletion(false));
  FileStatus [] files = fs.listStatus(testDir);
  assertTrue(files.length > 0);

  //check output file num and size.
  for (byte[] family : FAMILIES) {
    long kvCount= 0;
    RemoteIterator<LocatedFileStatus> iterator =
            fs.listFiles(testDir.suffix("/" + new String(family)), true);
    while (iterator.hasNext()) {
      LocatedFileStatus keyFileStatus = iterator.next();
      HFile.Reader reader =
              HFile.createReader(fs, keyFileStatus.getPath(), new CacheConfig(conf), true, conf);
      HFileScanner scanner = reader.getScanner(false, false, false);

      kvCount += reader.getEntries();
      scanner.seekTo();
      long perKVSize = scanner.getCell().getSerializedSize();
      assertTrue("Data size of each file should not be too large.",
              perKVSize * reader.getEntries() <= hregionMaxFilesize);
    }
    assertEquals("Should write expected data in output file.", ROWSPERSPLIT, kvCount);
  }
}
 

/**
 * Injects errors into the pread calls of an on-disk file, and makes
 * sure those bubble up to the HFile scanner
 */
@Test
public void testHFileScannerThrowsErrors() throws IOException {
  Path hfilePath = new Path(new Path(
      util.getDataTestDir("internalScannerExposesErrors"),
      "regionname"), "familyname");
  HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
  FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
  FileSystem fs = new HFileSystem(faultyfs);
  CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
  HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
  StoreFileWriter writer = new StoreFileWriter.Builder(
      util.getConfiguration(), cacheConf, hfs)
          .withOutputDir(hfilePath)
          .withFileContext(meta)
          .build();
  TestHStoreFile.writeStoreFile(
      writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));

  HStoreFile sf = new HStoreFile(fs, writer.getPath(), util.getConfiguration(), cacheConf,
      BloomType.NONE, true);
  sf.initReader();
  StoreFileReader reader = sf.getReader();
  HFileScanner scanner = reader.getScanner(false, true);

  FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
  assertNotNull(inStream);

  scanner.seekTo();
  // Do at least one successful read
  assertTrue(scanner.next());

  faultyfs.startFaults();

  try {
    int scanned=0;
    while (scanner.next()) {
      scanned++;
    }
    fail("Scanner didn't throw after faults injected");
  } catch (IOException ioe) {
    LOG.info("Got expected exception", ioe);
    assertTrue(ioe.getMessage().contains("Fault"));
  }
  reader.close(true); // end of test so evictOnClose
}
 

/**
 * This test creates an hfile and then the dir structures and files to verify that references to
 * hfilelinks (created by snapshot clones) can be properly interpreted.
 */
@Test
public void testReferenceToHFileLink() throws IOException {
  // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
  Configuration testConf = new Configuration(this.conf);
  CommonFSUtils.setRootDir(testConf, testDir);

  // adding legal table name chars to verify regex handles it.
  RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("_original-evil-name")).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);

  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
  StoreFileWriter writer = new StoreFileWriter.Builder(testConf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);
  Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());

  // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
  RegionInfo hriClone = RegionInfoBuilder.newBuilder(TableName.valueOf("clone")).build();
  HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
    CommonFSUtils.getTableDir(testDir, hri.getTable()), hriClone);
  Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);
  HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
  Path linkFilePath =
    new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));

  // create splits of the link.
  // <root>/clone/splitA/<cf>/<reftohfilelink>,
  // <root>/clone/splitB/<cf>/<reftohfilelink>
  RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build();
  RegionInfo splitHriB =
    RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build();
  HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true);
  f.initReader();
  Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top
  Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom
  f.closeStoreFile(true);
  // OK test the thing
  CommonFSUtils.logFileSystemState(fs, testDir, LOG);

  // There is a case where a file with the hfilelink pattern is actually a daughter
  // reference to a hfile link. This code in StoreFile that handles this case.

  // Try to open store file from link
  HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true);
  hsfA.initReader();

  // Now confirm that I can read from the ref to link
  int count = 1;
  HFileScanner s = hsfA.getReader().getScanner(false, false);
  s.seekTo();
  while (s.next()) {
    count++;
  }
  assertTrue(count > 0); // read some rows here

  // Try to open store file from link
  HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true);
  hsfB.initReader();

  // Now confirm that I can read from the ref to link
  HFileScanner sB = hsfB.getReader().getScanner(false, false);
  sB.seekTo();

  // count++ as seekTo() will advance the scanner
  count++;
  while (sB.next()) {
    count++;
  }

  // read the rest of the rows
  assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}
 

/**
 * Test that our mechanism of writing store files in one region to reference store files in other
 * regions works.
 */
@Test
public void testReference() throws IOException {
  final RegionInfo hri =
    RegionInfoBuilder.newBuilder(TableName.valueOf("testReferenceTb")).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
    new Path(testDir, hri.getTable().getNameAsString()), hri);

  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)
    .withFilePath(regionFs.createTempName()).withFileContext(meta).build();
  writeStoreFile(writer);

  Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
  HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);
  hsf.initReader();
  StoreFileReader reader = hsf.getReader();
  // Split on a row, not in middle of row. Midkey returned by reader
  // may be in middle of row. Create new one with empty column and
  // timestamp.
  byte[] midRow = CellUtil.cloneRow(reader.midKey().get());
  byte[] finalRow = CellUtil.cloneRow(reader.getLastKey().get());
  hsf.closeStoreFile(true);

  // Make a reference
  RegionInfo splitHri = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(midRow).build();
  Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true);
  HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true);
  refHsf.initReader();
  // Now confirm that I can read from the reference and that it only gets
  // keys from top half of the file.
  HFileScanner s = refHsf.getReader().getScanner(false, false);
  Cell kv = null;
  for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
    ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey());
    kv = KeyValueUtil.createKeyValueFromKey(bb);
    if (first) {
      assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0,
        midRow.length));
      first = false;
    }
  }
  assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0,
    finalRow.length));
}
 

@Test
public void testWrite() throws Exception {
  Configuration conf = new Configuration();
  HColumnDescriptor columnDescriptor = new HColumnDescriptor();
  // Disable block cache to ensure it reads the actual file content.
  columnDescriptor.setBlockCacheEnabled(false);
  FileSystem fs = FileSystem.get(conf);
  int blockSize = conf.getInt(Constants.HFILE_BLOCKSIZE, 16384);
  final StoreFile.Writer writer =
      new StoreFile.WriterBuilder(conf, new CacheConfig(conf, columnDescriptor), fs, blockSize)
          .withFilePath(new Path(tempFile.toURI()))
          .build();
  /* Create our RecordWriter */
  RecordWriter<BytesWritable, BytesWritable> hfileWriter =
      new HFileRecordWriter(writer);

  List<String> keys = Lists.newArrayList();
  List<String> values = Lists.newArrayList();
  for (int i = 0; i < 100; ++i) {
    String key = String.format("%03d", i);
    String val = "value " + i;
    keys.add(key);
    values.add(val);
    hfileWriter.write(new BytesWritable(key.getBytes()), new BytesWritable(val.getBytes()));
  }
  /* This internally closes the StoreFile.Writer */
  hfileWriter.close(null);

  HFile.Reader reader = HFile.createReader(fs, new Path(tempFile.toURI()),
      new CacheConfig(conf, columnDescriptor));
  HFileScanner scanner = reader.getScanner(false, false, false);
  boolean valid = scanner.seekTo();
  List<String> gotKeys = Lists.newArrayListWithCapacity(keys.size());
  List<String> gotValues = Lists.newArrayListWithCapacity(values.size());
  while(valid) {
    KeyValue keyValue = scanner.getKeyValue();
    gotKeys.add(new String(keyValue.getRow()));
    gotValues.add(new String(keyValue.getValue()));
    valid = scanner.next();
  }
  assertEquals(keys, gotKeys);
  assertEquals(values, gotValues);
  reader.close();
}