/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.OptionalLong; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.base.Joiner; import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; /** * Test HStoreFile */ @Category({ RegionServerTests.class, MediumTests.class }) public class TestHStoreFile { @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestHStoreFile.class); private static final Logger LOG = LoggerFactory.getLogger(TestHStoreFile.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); private static String ROOT_DIR = TEST_UTIL.getDataTestDir("TestStoreFile").toString(); private static final ChecksumType CKTYPE = ChecksumType.CRC32C; private static final int CKBYTES = 512; private static String TEST_FAMILY = "cf"; private static final char FIRST_CHAR = 'a'; private static final char LAST_CHAR = 'z'; @Rule public TestName name = new TestName(); private Configuration conf; private Path testDir; private FileSystem fs; @Before public void setUp() throws IOException { conf = TEST_UTIL.getConfiguration(); testDir = TEST_UTIL.getDataTestDir(name.getMethodName()); fs = testDir.getFileSystem(conf); } @AfterClass public static void tearDownAfterClass() { TEST_UTIL.cleanupTestDir(); } /** * Write a file and then assert that we can read from top and bottom halves using two * HalfMapFiles. */ @Test public void testBasicHalfMapFile() throws Exception { final RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("testBasicHalfMapFileTb")).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build(); StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(regionFs.createTempName()).withFileContext(meta).build(); writeStoreFile(writer); Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); HStoreFile sf = new HStoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE, true); checkHalfHFile(regionFs, sf); } private void writeStoreFile(final StoreFileWriter writer) throws IOException { writeStoreFile(writer, Bytes.toBytes(name.getMethodName()), Bytes.toBytes(name.getMethodName())); } // pick an split point (roughly halfway) byte[] SPLITKEY = new byte[] { (LAST_CHAR + FIRST_CHAR) / 2, FIRST_CHAR }; /* * Writes HStoreKey and ImmutableBytes data to passed writer and then closes it. * @param writer * @throws IOException */ public static void writeStoreFile(final StoreFileWriter writer, byte[] fam, byte[] qualifier) throws IOException { long now = System.currentTimeMillis(); try { for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) { for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) { byte[] b = new byte[] { (byte) d, (byte) e }; writer.append(new KeyValue(b, fam, qualifier, now, b)); } } } finally { writer.close(); } } /** * Test that our mechanism of writing store files in one region to reference store files in other * regions works. */ @Test public void testReference() throws IOException { final RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("testReferenceTb")).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(regionFs.createTempName()).withFileContext(meta).build(); writeStoreFile(writer); Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); StoreFileReader reader = hsf.getReader(); // Split on a row, not in middle of row. Midkey returned by reader // may be in middle of row. Create new one with empty column and // timestamp. byte[] midRow = CellUtil.cloneRow(reader.midKey().get()); byte[] finalRow = CellUtil.cloneRow(reader.getLastKey().get()); hsf.closeStoreFile(true); // Make a reference RegionInfo splitHri = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(midRow).build(); Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true); HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true); refHsf.initReader(); // Now confirm that I can read from the reference and that it only gets // keys from top half of the file. HFileScanner s = refHsf.getReader().getScanner(false, false); Cell kv = null; for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) { ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey()); kv = KeyValueUtil.createKeyValueFromKey(bb); if (first) { assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0, midRow.length)); first = false; } } assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0, finalRow.length)); } @Test public void testStoreFileReference() throws Exception { final RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("testStoreFileReference")).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(regionFs.createTempName()).withFileContext(meta).build(); writeStoreFile(writer); Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); writer.close(); HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true); file.initReader(); StoreFileReader r = file.getReader(); assertNotNull(r); StoreFileScanner scanner = new StoreFileScanner(r, mock(HFileScanner.class), false, false, 0, 0, false); // Verify after instantiating scanner refCount is increased assertTrue("Verify file is being referenced", file.isReferencedInReads()); scanner.close(); // Verify after closing scanner refCount is decreased assertFalse("Verify file is not being referenced", file.isReferencedInReads()); } @Test public void testEmptyStoreFileRestrictKeyRanges() throws Exception { StoreFileReader reader = mock(StoreFileReader.class); HStore store = mock(HStore.class); byte[] cf = Bytes.toBytes("ty"); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(cf); when(store.getColumnFamilyDescriptor()).thenReturn(cfd); try (StoreFileScanner scanner = new StoreFileScanner(reader, mock(HFileScanner.class), false, false, 0, 0, true)) { Scan scan = new Scan(); scan.setColumnFamilyTimeRange(cf, 0, 1); assertFalse(scanner.shouldUseScanner(scan, store, 0)); } } @Test public void testHFileLink() throws IOException { final RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build(); // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ Configuration testConf = new Configuration(this.conf); CommonFSUtils.setRootDir(testConf, testDir); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(regionFs.createTempName()).withFileContext(meta).build(); writeStoreFile(writer); Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY)); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName())); // Try to open store file from link StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true); HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf); assertTrue(storeFileInfo.isLink()); hsf.initReader(); // Now confirm that I can read from the link int count = 1; HFileScanner s = hsf.getReader().getScanner(false, false); s.seekTo(); while (s.next()) { count++; } assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count); } /** * This test creates an hfile and then the dir structures and files to verify that references to * hfilelinks (created by snapshot clones) can be properly interpreted. */ @Test public void testReferenceToHFileLink() throws IOException { // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ Configuration testConf = new Configuration(this.conf); CommonFSUtils.setRootDir(testConf, testDir); // adding legal table name chars to verify regex handles it. RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("_original-evil-name")).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file> StoreFileWriter writer = new StoreFileWriter.Builder(testConf, cacheConf, this.fs) .withFilePath(regionFs.createTempName()).withFileContext(meta).build(); writeStoreFile(writer); Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table> RegionInfo hriClone = RegionInfoBuilder.newBuilder(TableName.valueOf("clone")).build(); HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hriClone); Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName())); // create splits of the link. // <root>/clone/splitA/<cf>/<reftohfilelink>, // <root>/clone/splitB/<cf>/<reftohfilelink> RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build(); RegionInfo splitHriB = RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build(); HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true); f.initReader(); Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom f.closeStoreFile(true); // OK test the thing CommonFSUtils.logFileSystemState(fs, testDir, LOG); // There is a case where a file with the hfilelink pattern is actually a daughter // reference to a hfile link. This code in StoreFile that handles this case. // Try to open store file from link HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true); hsfA.initReader(); // Now confirm that I can read from the ref to link int count = 1; HFileScanner s = hsfA.getReader().getScanner(false, false); s.seekTo(); while (s.next()) { count++; } assertTrue(count > 0); // read some rows here // Try to open store file from link HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true); hsfB.initReader(); // Now confirm that I can read from the ref to link HFileScanner sB = hsfB.getReader().getScanner(false, false); sB.seekTo(); // count++ as seekTo() will advance the scanner count++; while (sB.next()) { count++; } // read the rest of the rows assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count); } private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f) throws IOException { f.initReader(); Cell midkey = f.getReader().midKey().get(); KeyValue midKV = (KeyValue) midkey; byte[] midRow = CellUtil.cloneRow(midKV); // Create top split. RegionInfo topHri = RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable()).setEndKey(SPLITKEY).build(); Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true); // Create bottom split. RegionInfo bottomHri = RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable()) .setStartKey(SPLITKEY).build(); Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false); // Make readers on top and bottom. HStoreFile topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true); topF.initReader(); StoreFileReader top = topF.getReader(); HStoreFile bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true); bottomF.initReader(); StoreFileReader bottom = bottomF.getReader(); ByteBuffer previous = null; LOG.info("Midkey: " + midKV.toString()); ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midKV.getKey()); try { // Now make two HalfMapFiles and assert they can read the full backing // file, one from the top and the other from the bottom. // Test bottom half first. // Now test reading from the top. boolean first = true; ByteBuffer key = null; HFileScanner topScanner = top.getScanner(false, false); while ((!topScanner.isSeeked() && topScanner.seekTo()) || (topScanner.isSeeked() && topScanner.next())) { key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey()); if ((PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(), key.arrayOffset(), key.limit())) > 0) { fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + midkey); } if (first) { first = false; LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key))); } } LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key))); first = true; HFileScanner bottomScanner = bottom.getScanner(false, false); while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) { previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey()); key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey()); if (first) { first = false; LOG.info("First in bottom: " + Bytes.toString(Bytes.toBytes(previous))); } assertTrue(key.compareTo(bbMidkeyBytes) < 0); } if (previous != null) { LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous))); } // Remove references. regionFs.cleanupDaughterRegion(topHri); regionFs.cleanupDaughterRegion(bottomHri); // Next test using a midkey that does not exist in the file. // First, do a key that is < than first key. Ensure splits behave // properly. byte[] badmidkey = Bytes.toBytes(" ."); assertTrue(fs.exists(f.getPath())); topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true); bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false); assertNull(bottomPath); topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true); topF.initReader(); top = topF.getReader(); // Now read from the top. first = true; topScanner = top.getScanner(false, false); KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue(); while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) { key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey()); keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit()); assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV, badmidkey, 0, badmidkey.length) >= 0); if (first) { first = false; KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key); LOG.info("First top when key < bottom: " + keyKV); String tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()); for (int i = 0; i < tmp.length(); i++) { assertTrue(tmp.charAt(i) == 'a'); } } } KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key); LOG.info("Last top when key < bottom: " + keyKV); String tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()); for (int i = 0; i < tmp.length(); i++) { assertTrue(tmp.charAt(i) == 'z'); } // Remove references. regionFs.cleanupDaughterRegion(topHri); regionFs.cleanupDaughterRegion(bottomHri); // Test when badkey is > than last key in file ('||' > 'zz'). badmidkey = Bytes.toBytes("|||"); topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true); bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false); assertNull(topPath); bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true); bottomF.initReader(); bottom = bottomF.getReader(); first = true; bottomScanner = bottom.getScanner(false, false); while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) { key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey()); if (first) { first = false; keyKV = KeyValueUtil.createKeyValueFromKey(key); LOG.info("First bottom when key > top: " + keyKV); tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()); for (int i = 0; i < tmp.length(); i++) { assertTrue(tmp.charAt(i) == 'a'); } } } keyKV = KeyValueUtil.createKeyValueFromKey(key); LOG.info("Last bottom when key > top: " + keyKV); for (int i = 0; i < tmp.length(); i++) { assertTrue(Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()) .charAt(i) == 'z'); } } finally { if (top != null) { top.close(true); // evict since we are about to delete the file } if (bottom != null) { bottom.close(true); // evict since we are about to delete the file } fs.delete(f.getPath(), true); } } private static StoreFileScanner getStoreFileScanner(StoreFileReader reader, boolean cacheBlocks, boolean pread) { return reader.getStoreFileScanner(cacheBlocks, pread, false, 0, 0, false); } private static final String localFormatter = "%010d"; private void bloomWriteRead(StoreFileWriter writer, FileSystem fs) throws Exception { float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0); Path f = writer.getPath(); long now = System.currentTimeMillis(); for (int i = 0; i < 2000; i += 2) { String row = String.format(localFormatter, i); KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col"), now, Bytes.toBytes("value")); writer.append(kv); } writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); HFileInfo fileInfo = new HFileInfo(context, conf); StoreFileReader reader = new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf); fileInfo.initMetaAndIndex(reader.getHFileReader()); reader.loadFileInfo(); reader.loadBloomfilter(); StoreFileScanner scanner = getStoreFileScanner(reader, false, false); // check false positives rate int falsePos = 0; int falseNeg = 0; for (int i = 0; i < 2000; i++) { String row = String.format(localFormatter, i); TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR); columns.add(Bytes.toBytes("family:col")); Scan scan = new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true); scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("family:col")); HStore store = mock(HStore.class); when(store.getColumnFamilyDescriptor()) .thenReturn(ColumnFamilyDescriptorBuilder.of("family")); boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE); if (i % 2 == 0) { if (!exists) { falseNeg++; } } else { if (exists) { falsePos++; } } } reader.close(true); // evict because we are about to delete the file fs.delete(f, true); assertEquals("False negatives: " + falseNeg, 0, falseNeg); int maxFalsePos = (int) (2 * 2000 * err); assertTrue("Too many false positives: " + falsePos + " (err=" + err + ", expected no more than " + maxFalsePos + ")", falsePos <= maxFalsePos); } private static final int BLOCKSIZE_SMALL = 8192; @Test public void testBloomFilter() throws Exception { conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01); conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); // write the file Path f = new Path(ROOT_DIR, name.getMethodName()); HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL) .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f) .withBloomType(BloomType.ROW).withMaxKeyCount(2000).withFileContext(meta).build(); bloomWriteRead(writer, fs); } @Test public void testDeleteFamilyBloomFilter() throws Exception { conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01); conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0); // write the file Path f = new Path(ROOT_DIR, name.getMethodName()); HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL) .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f) .withMaxKeyCount(2000).withFileContext(meta).build(); // add delete family long now = System.currentTimeMillis(); for (int i = 0; i < 2000; i += 2) { String row = String.format(localFormatter, i); KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col"), now, KeyValue.Type.DeleteFamily, Bytes.toBytes("value")); writer.append(kv); } writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); HFileInfo fileInfo = new HFileInfo(context, conf); StoreFileReader reader = new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf); fileInfo.initMetaAndIndex(reader.getHFileReader()); reader.loadFileInfo(); reader.loadBloomfilter(); // check false positives rate int falsePos = 0; int falseNeg = 0; for (int i = 0; i < 2000; i++) { String row = String.format(localFormatter, i); byte[] rowKey = Bytes.toBytes(row); boolean exists = reader.passesDeleteFamilyBloomFilter(rowKey, 0, rowKey.length); if (i % 2 == 0) { if (!exists) { falseNeg++; } } else { if (exists) { falsePos++; } } } assertEquals(1000, reader.getDeleteFamilyCnt()); reader.close(true); // evict because we are about to delete the file fs.delete(f, true); assertEquals("False negatives: " + falseNeg, 0, falseNeg); int maxFalsePos = (int) (2 * 2000 * err); assertTrue("Too many false positives: " + falsePos + " (err=" + err + ", expected no more than " + maxFalsePos, falsePos <= maxFalsePos); } /** * Test for HBASE-8012 */ @Test public void testReseek() throws Exception { // write the file Path f = new Path(ROOT_DIR, name.getMethodName()); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f) .withFileContext(meta).build(); writeStoreFile(writer); writer.close(); ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build(); HFileInfo fileInfo = new HFileInfo(context, conf); StoreFileReader reader = new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf); fileInfo.initMetaAndIndex(reader.getHFileReader()); // Now do reseek with empty KV to position to the beginning of the file KeyValue k = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY); StoreFileScanner s = getStoreFileScanner(reader, false, false); s.reseek(k); assertNotNull("Intial reseek should position at the beginning of the file", s.peek()); } @Test public void testBloomTypes() throws Exception { float err = (float) 0.01; FileSystem fs = FileSystem.getLocal(conf); conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err); conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); int rowCount = 50; int colCount = 10; int versions = 2; // run once using columns and once using rows BloomType[] bt = { BloomType.ROWCOL, BloomType.ROW }; int[] expKeys = { rowCount * colCount, rowCount }; // below line deserves commentary. it is expected bloom false positives // column = rowCount*2*colCount inserts // row-level = only rowCount*2 inserts, but failures will be magnified by // 2nd for loop for every column (2*colCount) float[] expErr = { 2 * rowCount * colCount * err, 2 * rowCount * 2 * colCount * err }; for (int x : new int[] { 0, 1 }) { // write the file Path f = new Path(ROOT_DIR, name.getMethodName() + x); HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL) .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f) .withBloomType(bt[x]).withMaxKeyCount(expKeys[x]).withFileContext(meta).build(); long now = System.currentTimeMillis(); for (int i = 0; i < rowCount * 2; i += 2) { // rows for (int j = 0; j < colCount * 2; j += 2) { // column qualifiers String row = String.format(localFormatter, i); String col = String.format(localFormatter, j); for (int k = 0; k < versions; ++k) { // versions KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col" + col), now - k, Bytes.toBytes(-1L)); writer.append(kv); } } } writer.close(); ReaderContext context = new ReaderContextBuilder().withFilePath(f).withFileSize(fs.getFileStatus(f).getLen()) .withFileSystem(fs).withInputStreamWrapper(new FSDataInputStreamWrapper(fs, f)).build(); HFileInfo fileInfo = new HFileInfo(context, conf); StoreFileReader reader = new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf); fileInfo.initMetaAndIndex(reader.getHFileReader()); reader.loadFileInfo(); reader.loadBloomfilter(); StoreFileScanner scanner = getStoreFileScanner(reader, false, false); assertEquals(expKeys[x], reader.getGeneralBloomFilter().getKeyCount()); HStore store = mock(HStore.class); when(store.getColumnFamilyDescriptor()) .thenReturn(ColumnFamilyDescriptorBuilder.of("family")); // check false positives rate int falsePos = 0; int falseNeg = 0; for (int i = 0; i < rowCount * 2; ++i) { // rows for (int j = 0; j < colCount * 2; ++j) { // column qualifiers String row = String.format(localFormatter, i); String col = String.format(localFormatter, j); TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR); columns.add(Bytes.toBytes("col" + col)); Scan scan = new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true); scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes(("col" + col))); boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE); boolean shouldRowExist = i % 2 == 0; boolean shouldColExist = j % 2 == 0; shouldColExist = shouldColExist || bt[x] == BloomType.ROW; if (shouldRowExist && shouldColExist) { if (!exists) { falseNeg++; } } else { if (exists) { falsePos++; } } } } reader.close(true); // evict because we are about to delete the file fs.delete(f, true); System.out.println(bt[x].toString()); System.out.println(" False negatives: " + falseNeg); System.out.println(" False positives: " + falsePos); assertEquals(0, falseNeg); assertTrue(falsePos < 2 * expErr[x]); } } @Test public void testSeqIdComparator() { assertOrdering(StoreFileComparators.SEQ_ID, mockStoreFile(true, 100, 1000, -1, "/foo/123"), mockStoreFile(true, 100, 1000, -1, "/foo/124"), mockStoreFile(true, 99, 1000, -1, "/foo/126"), mockStoreFile(true, 98, 2000, -1, "/foo/126"), mockStoreFile(false, 3453, -1, 1, "/foo/1"), mockStoreFile(false, 2, -1, 3, "/foo/2"), mockStoreFile(false, 1000, -1, 5, "/foo/2"), mockStoreFile(false, 76, -1, 5, "/foo/3")); } /** * Assert that the given comparator orders the given storefiles in the same way that they're * passed. */ private void assertOrdering(Comparator<? super HStoreFile> comparator, HStoreFile... sfs) { ArrayList<HStoreFile> sorted = Lists.newArrayList(sfs); Collections.shuffle(sorted); Collections.sort(sorted, comparator); LOG.debug("sfs: " + Joiner.on(",").join(sfs)); LOG.debug("sorted: " + Joiner.on(",").join(sorted)); assertTrue(Iterables.elementsEqual(Arrays.asList(sfs), sorted)); } /** * Create a mock StoreFile with the given attributes. */ private HStoreFile mockStoreFile(boolean bulkLoad, long size, long bulkTimestamp, long seqId, String path) { HStoreFile mock = Mockito.mock(HStoreFile.class); StoreFileReader reader = Mockito.mock(StoreFileReader.class); Mockito.doReturn(size).when(reader).length(); Mockito.doReturn(reader).when(mock).getReader(); Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult(); Mockito.doReturn(OptionalLong.of(bulkTimestamp)).when(mock).getBulkLoadTimestamp(); Mockito.doReturn(seqId).when(mock).getMaxSequenceId(); Mockito.doReturn(new Path(path)).when(mock).getPath(); String name = "mock storefile, bulkLoad=" + bulkLoad + " bulkTimestamp=" + bulkTimestamp + " seqId=" + seqId + " path=" + path; Mockito.doReturn(name).when(mock).toString(); return mock; } /** * Generate a list of KeyValues for testing based on given parameters * @return the rows key-value list */ List<KeyValue> getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier, byte[] family) { List<KeyValue> kvList = new ArrayList<>(); for (int i = 1; i <= numRows; i++) { byte[] b = Bytes.toBytes(i); LOG.info(Bytes.toString(b)); LOG.info(Bytes.toString(b)); for (long timestamp : timestamps) { kvList.add(new KeyValue(b, family, qualifier, timestamp, b)); } } return kvList; } /** * Test to ensure correctness when using StoreFile with multiple timestamps */ @Test public void testMultipleTimestamps() throws IOException { byte[] family = Bytes.toBytes("familyname"); byte[] qualifier = Bytes.toBytes("qualifier"); int numRows = 10; long[] timestamps = new long[] { 20, 10, 5, 1 }; Scan scan = new Scan(); // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. Path storedir = new Path(new Path(testDir, "7e0102"), Bytes.toString(family)); Path dir = new Path(storedir, "1234567890"); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withOutputDir(dir).withFileContext(meta).build(); List<KeyValue> kvList = getKeyValueSet(timestamps, numRows, qualifier, family); for (KeyValue kv : kvList) { writer.append(kv); } writer.appendMetadata(0, false); writer.close(); HStoreFile hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); HStore store = mock(HStore.class); when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of(family)); hsf.initReader(); StoreFileReader reader = hsf.getReader(); StoreFileScanner scanner = getStoreFileScanner(reader, false, false); TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR); columns.add(qualifier); scan.setTimeRange(20, 100); assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE)); scan.setTimeRange(1, 2); assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE)); scan.setTimeRange(8, 10); assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE)); // lets make sure it still works with column family time ranges scan.setColumnFamilyTimeRange(family, 7, 50); assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE)); // This test relies on the timestamp range optimization scan = new Scan(); scan.setTimeRange(27, 50); assertTrue(!scanner.shouldUseScanner(scan, store, Long.MIN_VALUE)); // should still use the scanner because we override the family time range scan = new Scan(); scan.setTimeRange(27, 50); scan.setColumnFamilyTimeRange(family, 7, 50); assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE)); } @Test public void testCacheOnWriteEvictOnClose() throws Exception { Configuration conf = this.conf; // Find a home for our files (regiondir ("7e0102") and familyname). Path baseDir = new Path(new Path(testDir, "7e0102"), "twoCOWEOC"); // Grab the block cache and get the initial hit/miss counts BlockCache bc = BlockCacheFactory.createBlockCache(conf); assertNotNull(bc); CacheStats cs = bc.getStats(); long startHit = cs.getHitCount(); long startMiss = cs.getMissCount(); long startEvicted = cs.getEvictedCount(); // Let's write a StoreFile with three blocks, with cache on write off conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); CacheConfig cacheConf = new CacheConfig(conf, bc); Path pathCowOff = new Path(baseDir, "123456789"); StoreFileWriter writer = writeStoreFile(conf, cacheConf, pathCowOff, 3); HStoreFile hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); LOG.debug(hsf.getPath().toString()); // Read this file, we should see 3 misses hsf.initReader(); StoreFileReader reader = hsf.getReader(); reader.loadFileInfo(); StoreFileScanner scanner = getStoreFileScanner(reader, true, true); scanner.seek(KeyValue.LOWESTKEY); while (scanner.next() != null) { continue; } assertEquals(startHit, cs.getHitCount()); assertEquals(startMiss + 3, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); startMiss += 3; scanner.close(); reader.close(cacheConf.shouldEvictOnClose()); // Now write a StoreFile with three blocks, with cache on write on conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); cacheConf = new CacheConfig(conf, bc); Path pathCowOn = new Path(baseDir, "123456788"); writer = writeStoreFile(conf, cacheConf, pathCowOn, 3); hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); // Read this file, we should see 3 hits hsf.initReader(); reader = hsf.getReader(); scanner = getStoreFileScanner(reader, true, true); scanner.seek(KeyValue.LOWESTKEY); while (scanner.next() != null) { continue; } assertEquals(startHit + 3, cs.getHitCount()); assertEquals(startMiss, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); startHit += 3; scanner.close(); reader.close(cacheConf.shouldEvictOnClose()); // Let's read back the two files to ensure the blocks exactly match hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); StoreFileReader readerOne = hsf.getReader(); readerOne.loadFileInfo(); StoreFileScanner scannerOne = getStoreFileScanner(readerOne, true, true); scannerOne.seek(KeyValue.LOWESTKEY); hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); StoreFileReader readerTwo = hsf.getReader(); readerTwo.loadFileInfo(); StoreFileScanner scannerTwo = getStoreFileScanner(readerTwo, true, true); scannerTwo.seek(KeyValue.LOWESTKEY); Cell kv1 = null; Cell kv2 = null; while ((kv1 = scannerOne.next()) != null) { kv2 = scannerTwo.next(); assertTrue(kv1.equals(kv2)); KeyValue keyv1 = KeyValueUtil.ensureKeyValue(kv1); KeyValue keyv2 = KeyValueUtil.ensureKeyValue(kv2); assertTrue(Bytes.compareTo(keyv1.getBuffer(), keyv1.getKeyOffset(), keyv1.getKeyLength(), keyv2.getBuffer(), keyv2.getKeyOffset(), keyv2.getKeyLength()) == 0); assertTrue(Bytes.compareTo(kv1.getValueArray(), kv1.getValueOffset(), kv1.getValueLength(), kv2.getValueArray(), kv2.getValueOffset(), kv2.getValueLength()) == 0); } assertNull(scannerTwo.next()); assertEquals(startHit + 6, cs.getHitCount()); assertEquals(startMiss, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); startHit += 6; scannerOne.close(); readerOne.close(cacheConf.shouldEvictOnClose()); scannerTwo.close(); readerTwo.close(cacheConf.shouldEvictOnClose()); // Let's close the first file with evict on close turned on conf.setBoolean("hbase.rs.evictblocksonclose", true); cacheConf = new CacheConfig(conf, bc); hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); reader = hsf.getReader(); reader.close(cacheConf.shouldEvictOnClose()); // We should have 3 new evictions but the evict count stat should not change. Eviction because // of HFile invalidation is not counted along with normal evictions assertEquals(startHit, cs.getHitCount()); assertEquals(startMiss, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); // Let's close the second file with evict on close turned off conf.setBoolean("hbase.rs.evictblocksonclose", false); cacheConf = new CacheConfig(conf, bc); hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); reader = hsf.getReader(); reader.close(cacheConf.shouldEvictOnClose()); // We expect no changes assertEquals(startHit, cs.getHitCount()); assertEquals(startMiss, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); } private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo hri, final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef) throws IOException { Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null); if (null == path) { return null; } Path regionDir = regionFs.commitDaughterRegion(hri); return new Path(new Path(regionDir, family), path.getName()); } private StoreFileWriter writeStoreFile(Configuration conf, CacheConfig cacheConf, Path path, int numBlocks) throws IOException { // Let's put ~5 small KVs in each block, so let's make 5*numBlocks KVs int numKVs = 5 * numBlocks; List<KeyValue> kvs = new ArrayList<>(numKVs); byte[] b = Bytes.toBytes("x"); int totalSize = 0; for (int i = numKVs; i > 0; i--) { KeyValue kv = new KeyValue(b, b, b, i, b); kvs.add(kv); // kv has memstoreTS 0, which takes 1 byte to store. totalSize += kv.getLength() + 1; } int blockSize = totalSize / numBlocks; HFileContext meta = new HFileContextBuilder().withBlockSize(blockSize).withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build(); // We'll write N-1 KVs to ensure we don't write an extra block kvs.remove(kvs.size() - 1); for (KeyValue kv : kvs) { writer.append(kv); } writer.appendMetadata(0, false); writer.close(); return writer; } /** * Check if data block encoding information is saved correctly in HFile's file info. */ @Test public void testDataBlockEncodingMetaData() throws IOException { // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. Path dir = new Path(new Path(testDir, "7e0102"), "familyname"); Path path = new Path(dir, "1234567890"); DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF; cacheConf = new CacheConfig(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build(); writer.close(); HStoreFile storeFile = new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true); storeFile.initReader(); StoreFileReader reader = storeFile.getReader(); Map<byte[], byte[]> fileInfo = reader.loadFileInfo(); byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING); assertArrayEquals(dataBlockEncoderAlgo.getNameInBytes(), value); } }