org.apache.hadoop.hbase.util.Bytes Java Examples

The following examples show how to use org.apache.hadoop.hbase.util.Bytes. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HelloHBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Invokes Admin#createNamespace and Admin#createTable to create a namespace
 * with a table that has one column-family.
 *
 * @param admin Standard Admin object
 * @throws IOException If IO problem encountered
 */
static void createNamespaceAndTable(final Admin admin) throws IOException {

  if (!namespaceExists(admin, MY_NAMESPACE_NAME)) {
    System.out.println("Creating Namespace [" + MY_NAMESPACE_NAME + "].");

    admin.createNamespace(NamespaceDescriptor
            .create(MY_NAMESPACE_NAME).build());
  }
  if (!admin.tableExists(MY_TABLE_NAME)) {
    System.out.println("Creating Table [" + MY_TABLE_NAME.getNameAsString()
            + "], with one Column Family ["
            + Bytes.toString(MY_COLUMN_FAMILY_NAME) + "].");

    admin.createTable(new TableDescriptorBuilder.ModifyableTableDescriptor(MY_TABLE_NAME)
      .setColumnFamily(
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
          MY_COLUMN_FAMILY_NAME)));
  }
}
 
Example #2
Source File: TestApplyAndFilterDeletesFilter.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
/**
 * Test that we don't cover other columns when we have a delete column.
 */
@Test
public void testDeleteColumnCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue d = createKvForType(Type.DeleteColumn, 12);
  byte[] qual2 = Bytes.add(qualifier, Bytes.toBytes("-other"));
  KeyValue put =
      new KeyValue(row, family, qual2, 0, qual2.length, 11, Type.Put, value, 0,
          value.length);

  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // different column put should still be visible
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // set a delete family, but in the past
  d = createKvForType(Type.DeleteFamily, 10);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // add back in the original delete column
  d = createKvForType(Type.DeleteColumn, 11);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // onto a different family, so that must be visible too
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));
}
 
Example #3
Source File: SpillMap.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private boolean canFit(byte[] curValue, byte[] newValue) {
    if (thresholdBytes < newValue.length) {
        // TODO resize page size if single element is too big,
        // Can this ever happen?
        throw new RuntimeException("page size too small to store a single KV element");
    }

    int resultSize = newValue.length + Bytes.SIZEOF_INT;
    if (curValue != null) {
        // Key existed before
        // Ensure to compensate for potential larger byte[] for agg
        resultSize = Math.max(0, resultSize - (curValue.length + Bytes.SIZEOF_INT));
    }

    if ((thresholdBytes - totalResultSize) <= (resultSize)) {
        // KV does not fit
        return false;
    }
    // KV fits
    return true;
}
 
Example #4
Source File: AbstractPrefixMatchingExtractor.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
@Override
public Collection<byte[]> extract(Result result) {
    List<byte[]> values = Lists.newArrayList();

    NavigableMap<byte[], byte[]> qualifiersToValues = result.getFamilyMap(columnFamily);
    if (qualifiersToValues != null) {
        for (byte[] qualifier : qualifiersToValues.navigableKeySet().tailSet(prefix)) {
            if (Bytes.startsWith(qualifier, prefix)) {
                values.add(extractInternal(qualifier, qualifiersToValues.get(qualifier)));
            } else {
                break;
            }
        }
    }
    return values;
}
 
Example #5
Source File: ScannerModel.java    From hbase with Apache License 2.0 6 votes vote down vote up
public ByteArrayComparableModel(
    ByteArrayComparable comparator) {
  String typeName = comparator.getClass().getSimpleName();
  ComparatorType type = ComparatorType.valueOf(typeName);
  this.type = typeName;
  switch (type) {
    case BinaryComparator:
    case BinaryPrefixComparator:
      this.value = Bytes.toString(Base64.getEncoder().encode(comparator.getValue()));
      break;
    case BitComparator:
      this.value = Bytes.toString(Base64.getEncoder().encode(comparator.getValue()));
      this.op = ((BitComparator)comparator).getOperator().toString();
      break;
    case NullComparator:
      break;
    case RegexStringComparator:
    case SubstringComparator:
      this.value = Bytes.toString(comparator.getValue());
      break;
    default:
      throw new RuntimeException("unhandled filter type: " + type);
  }
}
 
Example #6
Source File: TestTableRowkeyPair.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int expectedSignum) throws IOException {
    
    final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey));
    TableRowkeyPair pair1 = new TableRowkeyPair(aTable, arowkey);
    
    ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey));
    TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey);
    
    TableRowkeyPair.Comparator comparator = new TableRowkeyPair.Comparator();
    try( ByteArrayOutputStream baosA = new ByteArrayOutputStream();
         ByteArrayOutputStream baosB = new ByteArrayOutputStream()) {
        
        pair1.write(new DataOutputStream(baosA));
        pair2.write(new DataOutputStream(baosB));
        Assert.assertEquals(expectedSignum , signum(pair1.compareTo(pair2)));
        Assert.assertEquals(expectedSignum , signum(comparator.compare(baosA.toByteArray(), 0, baosA.size(), baosB.toByteArray(), 0, baosB.size())));
        Assert.assertEquals(expectedSignum, -signum(comparator.compare(baosB.toByteArray(), 0, baosB.size(), baosA.toByteArray(), 0, baosA.size())));
    }

}
 
Example #7
Source File: DistinctValueWithCountServerAggregator.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private int countMapHeapSize() {
    int size = 0;
    if (this.valueVsCount.size() > 0) {
        for (ImmutableBytesPtr key : this.valueVsCount.keySet()) {
            size += SizedUtil.MAP_ENTRY_SIZE + // entry
                    Bytes.SIZEOF_INT + // key size
                    key.getLength() + SizedUtil.ARRAY_SIZE; // value size
        }
    } else {
        // Initially when the getSize() is called, we dont have any entries in the map so as to
        // tell the exact heap need. Let us approximate the #entries
        SizedUtil.sizeOfMap(DEFAULT_ESTIMATED_DISTINCT_VALUES,
                SizedUtil.IMMUTABLE_BYTES_PTR_SIZE, Bytes.SIZEOF_INT);
    }
    return size;
}
 
Example #8
Source File: TestAutoFlush.java    From phoenix-omid with Apache License 2.0 6 votes vote down vote up
@Test(timeOut = 10_000)
public void testReadWithSeveralUncommitted(ITestContext context) throws Exception {

    byte[] family = Bytes.toBytes(TEST_FAMILY);
    byte[] row = Bytes.toBytes("row");
    byte[] col = Bytes.toBytes("col1");
    byte[] data = Bytes.toBytes("data");
    TransactionManager tm = newTransactionManager(context);
    TTable table = new TTable(connection, TEST_TABLE);

    // Turn off autoflush
    table.setAutoFlush(false);

    Transaction t = tm.begin();
    Put put = new Put(row);
    put.addColumn(family, col, data);
    table.put(t, put);

    // Data shouldn't be in DB yet
    Get get = new Get(row);
    Result result = table.getHTable().get(get);
    assertEquals(result.size(), 0, "Writes are already in DB");

    //data should be readable within same transaction
    result = table.get(t,get);
    assertEquals(result.size(), 1, "Writes should be read by same transaction");

    tm.commit(t);

    // After commit, both the cell and shadow cell should be there.
    // That's why we check for two elements in the test assertion
    result = table.getHTable().get(get);
    assertEquals(result.size(), 2, "Writes were not flushed to DB");
}
 
Example #9
Source File: MC.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
private void filterWithRsAndLocality(Set<byte[]> targets, String table) throws IOException {
    long startTimestamp = System.currentTimeMillis();
    Util.printVerboseMessage(args, Util.getMethodName() + " - start");

    Map<byte[], HRegionInfo> regionMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    String regex = (String) args.valueOf(Args.OPTION_REGION_SERVER);
    for (Map.Entry<HRegionInfo, ServerName> entry : getRegionLocations(table).entrySet()) {
        String serverName = entry.getValue().getHostname() + "," + entry.getValue().getPort();
        if (serverName.matches(regex)) {
            regionMap.put(entry.getKey().getRegionName(), entry.getKey());
            byte[] regionName = entry.getKey().getRegionName();
            targets.add(regionName);
            regionTableMap.put(regionName, table);
            regionRSMap.put(regionName, serverName);
        }
    }

    filterWithDataLocality(targets, regionMap);

    Util.printVerboseMessage(args, Util.getMethodName() + " - end", startTimestamp);
}
 
Example #10
Source File: TenantCacheTest.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testExpiresButStaysInPersistentAfterTimeout() throws Exception {
    int maxServerCacheTimeToLive = 100;
    int maxServerCachePersistenceTimeToLive = 1000;
    long maxBytes = 1000;
    GlobalMemoryManager memoryManager = new GlobalMemoryManager(maxBytes);
    ManualTicker ticker = new ManualTicker();
    TenantCacheImpl cache = new TenantCacheImpl(memoryManager, maxServerCacheTimeToLive, maxServerCachePersistenceTimeToLive, ticker);
    ImmutableBytesPtr cacheId1 = new ImmutableBytesPtr(Bytes.toBytes(1L));
    ImmutableBytesWritable cachePtr = new ImmutableBytesWritable(Bytes.toBytes("a"));
    cache.addServerCache(cacheId1, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory, true, true, MetaDataProtocol.PHOENIX_VERSION);
    assertEquals(maxBytes-1, memoryManager.getAvailableMemory());
    assertNotNull(cache.getServerCache(cacheId1));

    // Expire it from live cache but not persistent cache
    ticker.time += (maxServerCacheTimeToLive + 1) * 1000000;
    cache.cleanUp();
    assertEquals(maxBytes-1, memoryManager.getAvailableMemory());
    assertNotNull(cache.getServerCache(cacheId1));

    // Expire it from persistent cache as well
    ticker.time += (maxServerCachePersistenceTimeToLive + 1) * 1000000;
    cache.cleanUp();
    assertEquals(maxBytes, memoryManager.getAvailableMemory());
    assertNull(cache.getServerCache(cacheId1));
}
 
Example #11
Source File: TestHFileOutputFormat2.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void setupMockColumnFamiliesForCompression(Table table,
    Map<String, Compression.Algorithm> familyToCompression) throws IOException {

  TableDescriptorBuilder mockTableDescriptor =
    TableDescriptorBuilder.newBuilder(TABLE_NAMES[0]);
  for (Entry<String, Compression.Algorithm> entry : familyToCompression.entrySet()) {
    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(entry.getKey()))
      .setMaxVersions(1)
      .setCompressionType(entry.getValue())
      .setBlockCacheEnabled(false)
      .setTimeToLive(0)
      .build();

    mockTableDescriptor.setColumnFamily(columnFamilyDescriptor);
  }
  Mockito.doReturn(mockTableDescriptor.build()).when(table).getDescriptor();
}
 
Example #12
Source File: TestSequenceIdMonotonicallyIncreasing.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSplit()
    throws IOException, InterruptedException, ExecutionException, TimeoutException {
  try (Table table = createTable(false)) {
    table.put(new Put(Bytes.toBytes(0)).addColumn(CF, CQ, Bytes.toBytes(0)));
    table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(0)));
  }
  UTIL.flush(NAME);
  HRegionServer rs = UTIL.getRSForFirstRegionInTable(NAME);
  RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
  UTIL.getAdmin().splitRegionAsync(region.getRegionName(), Bytes.toBytes(1)).get(1,
    TimeUnit.MINUTES);
  long maxSeqId = getMaxSeqId(rs, region);
  RegionLocator locator = UTIL.getConnection().getRegionLocator(NAME);
  HRegionLocation locA = locator.getRegionLocation(Bytes.toBytes(0), true);
  HRegionLocation locB = locator.getRegionLocation(Bytes.toBytes(1), true);
  assertEquals(maxSeqId + 1, locA.getSeqNum());
  assertEquals(maxSeqId + 1, locB.getSeqNum());
}
 
Example #13
Source File: TestAtomicOperation.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void assertICV(byte [] row,
                       byte [] familiy,
                       byte[] qualifier,
                       long amount,
                       boolean fast) throws IOException {
  // run a get and see?
  Get get = new Get(row);
  if (fast) get.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
  get.addColumn(familiy, qualifier);
  Result result = region.get(get);
  assertEquals(1, result.size());

  Cell kv = result.rawCells()[0];
  long r = Bytes.toLong(CellUtil.cloneValue(kv));
  assertEquals(amount, r);
}
 
Example #14
Source File: LoggingConsumer.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.setBoolean("hbase.replication", true);

    ZooKeeperItf zk = ZkUtil.connect("localhost", 20000);
    SepModel sepModel = new SepModelImpl(zk, conf);

    final String subscriptionName = "logger";

    if (!sepModel.hasSubscription(subscriptionName)) {
        sepModel.addSubscriptionSilent(subscriptionName);
    }

    PayloadExtractor payloadExtractor = new BasePayloadExtractor(Bytes.toBytes("sep-user-demo"), Bytes.toBytes("info"),
            Bytes.toBytes("payload"));

    SepConsumer sepConsumer = new SepConsumer(subscriptionName, 0, new EventLogger(), 1, "localhost", zk, conf,
            payloadExtractor);

    sepConsumer.start();
    System.out.println("Started");

    while (true) {
        Thread.sleep(Long.MAX_VALUE);
    }
}
 
Example #15
Source File: AlterTableIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private void assertEncodedCQValue(String columnFamily, String columnName, String schemaName, String tableName, int expectedValue) throws Exception {
    String query = "SELECT " + COLUMN_QUALIFIER + " FROM \"SYSTEM\".CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME
            + " = ? " + " AND " + COLUMN_FAMILY + " = ?" + " AND " + COLUMN_NAME  + " = ?" + " AND " + COLUMN_QUALIFIER  + " IS NOT NULL";
    try (Connection conn = DriverManager.getConnection(getUrl())) {
        PreparedStatement stmt = conn.prepareStatement(query);
        stmt.setString(1, schemaName);
        stmt.setString(2, tableName);
        stmt.setString(3, columnFamily);
        stmt.setString(4, columnName);
        ResultSet rs = stmt.executeQuery();
        assertTrue(rs.next());
        if (columnEncoded) {
            assertTrue(Bytes.equals(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS.encode(expectedValue), rs.getBytes(1)));
        } else {
            assertTrue(Bytes.equals(columnName.getBytes(), rs.getBytes(1)));
        }
        assertFalse(rs.next());
    }
}
 
Example #16
Source File: HBCKMetaTableAccessor.java    From hbase-operator-tools with Apache License 2.0 6 votes vote down vote up
/**
 * Returns all regions in meta for the given table.
 * @param conn a valid, open connection.
 * @param table the table to list regions in meta.
 * @return a list of <code>RegionInfo</code> for all table regions present in meta.
 * @throws IOException on any issues related with scanning meta table
 */
public static List<RegionInfo> getTableRegions(final Connection conn, final TableName table)
    throws IOException {
  final MetaScanner<RegionInfo> scanner = new MetaScanner<>();
  final String startRow = Bytes.toString(table.getName()) + ",,";
  final String stopRow = Bytes.toString(table.getName()) + " ,,";
  return scanner.scanMeta(conn,
    scan -> {
      scan.withStartRow(Bytes.toBytes(startRow));
      scan.withStopRow(Bytes.toBytes(stopRow));
    },
    r -> {
      Cell cell = r.getColumnLatestCell(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
      if(cell != null) {
        RegionInfo info = RegionInfo
          .parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
        return info;
      }
      return null;
    });
}
 
Example #17
Source File: MultiEncodedCQKeyValueComparisonFilter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public void write(DataOutput output) throws IOException {
    try {
        WritableUtils.writeVInt(output, minQualifier);
        WritableUtils.writeVInt(output, maxQualifier);
        WritableUtils.writeVInt(output, whereExpressionMinQualifier);
        WritableUtils.writeVInt(output, whereExpressionMaxQualifier);
        WritableUtils.writeVInt(output, encodingScheme.ordinal());
        super.write(output);
        output.writeBoolean(allCFs);
        if (!allCFs) {
            Bytes.writeByteArray(output, essentialCF);
        }
    } catch (DoNotRetryIOException e) {
        throw e;
    } catch (Throwable t) { // Catches incompatibilities during reading/writing and doesn't retry
        ServerUtil.throwIOException("MultiEncodedCQKeyValueComparisonFilter failed during writing", t);
    }
}
 
Example #18
Source File: PhTypeUtil.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private static int encodeShort(short v, byte[] b, int o) {
    checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT);
    b[o + 0] = (byte) ((v >> 8) ^ 0x80); // Flip sign bit so that Short is
                                         // binary comparable
    b[o + 1] = (byte) v;
    return Bytes.SIZEOF_SHORT;
}
 
Example #19
Source File: AbstractIteratorRegionScanner.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
private boolean containedInScan(Cell kv) {
    byte[] rowArray = kv.getRowArray();
    int rowOffset = kv.getRowOffset();
    int rowLength = kv.getRowLength();
    if(Bytes.compareTo(scan.getStartRow(),0,scan.getStartRow().length,rowArray,rowOffset,rowLength)>0) return false;
    if(Bytes.compareTo(scan.getStopRow(),0,scan.getStopRow().length,rowArray,rowOffset,rowLength)<=0) return false;
    byte[] family = CellUtil.cloneFamily(kv);
    Map<byte[], NavigableSet<byte[]>> familyMap = scan.getFamilyMap();
    if(familyMap.size()<=0) return true;

    if(!familyMap.containsKey(family)) return false;
    NavigableSet<byte[]> qualifiersToFetch = familyMap.get(family);
    if(qualifiersToFetch.size()<=0) return true;
    return qualifiersToFetch.contains(CellUtil.cloneQualifier(kv));
}
 
Example #20
Source File: TestWALObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Coprocessors shouldn't get notice of empty waledits.
 */
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
  RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE));
  TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  WAL log = wals.getWAL(null);
  try {
    SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class);

    cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);

    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());

    final long now = EnvironmentEdgeManager.currentTime();
    long txid = log.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes),
      new WALEdit());
    log.sync(txid);

    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
  } finally {
    log.close();
  }
}
 
Example #21
Source File: TestMobFileCache.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  UTIL = new HBaseTestingUtility();
  conf = UTIL.getConfiguration();
  conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(UTIL.createTableDescriptor(
      TableName.valueOf("testMobFileCache"), ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS,
      3, HConstants.FOREVER, ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY1))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY2))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY3))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  mobFileCache = new MobFileCache(conf);
  region = HBaseTestingUtility
    .createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf, tableDescriptor, mobFileCache);
}
 
Example #22
Source File: TestBufferedMutator.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void verifyData() throws IOException {
  try (Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME)) {
    for (int i = 0; i < COUNT; i++) {
      Result r = table.get(new Get(Bytes.toBytes(i)));
      assertArrayEquals(VALUE, ((Result) r).getValue(CF, CQ));
    }
  }
}
 
Example #23
Source File: TestStoreScannerClosure.java    From hbase with Apache License 2.0 5 votes vote down vote up
NavigableSet<byte[]> getCols(String... strCols) {
  NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  for (String col : strCols) {
    byte[] bytes = Bytes.toBytes(col);
    cols.add(bytes);
  }
  return cols;
}
 
Example #24
Source File: TestHRegionReplayEvents.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testSeqIdsFromReplay() throws IOException {
  // test the case where seqId's coming from replayed WALEdits are made persisted with their
  // original seqIds and they are made visible through mvcc read point upon replay
  String method = name.getMethodName();
  byte[] tableName = Bytes.toBytes(method);
  byte[] family = Bytes.toBytes("family");

  HRegion region = initHRegion(tableName, family);
  try {
    // replay an entry that is bigger than current read point
    long readPoint = region.getMVCC().getReadPoint();
    long origSeqId = readPoint + 100;

    Put put = new Put(row).addColumn(family, row, row);
    put.setDurability(Durability.SKIP_WAL); // we replay with skip wal
    replay(region, put, origSeqId);

    // read point should have advanced to this seqId
    assertGet(region, family, row);

    // region seqId should have advanced at least to this seqId
    assertEquals(origSeqId, region.getReadPoint(null));

    // replay an entry that is smaller than current read point
    // caution: adding an entry below current read point might cause partial dirty reads. Normal
    // replay does not allow reads while replay is going on.
    put = new Put(row2).addColumn(family, row2, row2);
    put.setDurability(Durability.SKIP_WAL);
    replay(region, put, origSeqId - 50);

    assertGet(region, family, row2);
  } finally {
    region.close();
  }
}
 
Example #25
Source File: TestVisibilityLabelsWithDeletes.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void scanAll(Result[] next) throws IOException {
  CellScanner cellScanner = next[0].cellScanner();
  cellScanner.advance();
  Cell current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(127L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(126L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(125L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(124L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(123L, current.getTimestamp());
  cellScanner = next[1].cellScanner();
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row2, 0, row2.length));
}
 
Example #26
Source File: TestSimpleTotalOrderPartitioner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testSplit() throws Exception {
  String start = "a";
  String end = "{";
  SimpleTotalOrderPartitioner<byte []> p = new SimpleTotalOrderPartitioner<>();

  this.conf.set(SimpleTotalOrderPartitioner.START, start);
  this.conf.set(SimpleTotalOrderPartitioner.END, end);
  p.setConf(this.conf);
  ImmutableBytesWritable c = new ImmutableBytesWritable(Bytes.toBytes("c"));
  // If one reduce, partition should be 0.
  int partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 1);
  assertEquals(0, partition);
  // If two reduces, partition should be 0.
  partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(0, partition);
  // Divide in 3.
  partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(0, partition);
  ImmutableBytesWritable q = new ImmutableBytesWritable(Bytes.toBytes("q"));
  partition = p.getPartition(q, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(1, partition);
  partition = p.getPartition(q, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(2, partition);
  // What about end and start keys.
  ImmutableBytesWritable startBytes =
    new ImmutableBytesWritable(Bytes.toBytes(start));
  partition = p.getPartition(startBytes, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(0, partition);
  partition = p.getPartition(startBytes, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(0, partition);
  ImmutableBytesWritable endBytes =
    new ImmutableBytesWritable(Bytes.toBytes("z"));
  partition = p.getPartition(endBytes, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(1, partition);
  partition = p.getPartition(endBytes, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(2, partition);
}
 
Example #27
Source File: HBaseUtils.java    From PoseidonX with Apache License 2.0 5 votes vote down vote up
/**
 * 获得一个字符串的字节数组
 * @param value
 * @return
 */
public static byte[] getBytes(String value) {
	if (value == null) {
		return null;
	}
	return Bytes.toBytes(value);
}
 
Example #28
Source File: TestCombinedAsyncWriter.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void doTest(boolean withTrailer) throws IOException {
  int columnCount = 5;
  int recordCount = 5;
  TableName tableName = TableName.valueOf("tablename");
  byte[] row = Bytes.toBytes("row");
  long timestamp = System.currentTimeMillis();
  Path path1 = getPath(1);
  Path path2 = getPath(2);
  FileSystem fs = UTIL.getTestFileSystem();
  Configuration conf = UTIL.getConfiguration();
  try (
    AsyncWriter writer1 = AsyncFSWALProvider.createAsyncWriter(conf, fs, path1, false,
      EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
    AsyncWriter writer2 = AsyncFSWALProvider.createAsyncWriter(conf, fs, path2, false,
      EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
    CombinedAsyncWriter writer = CombinedAsyncWriter.create(writer1, writer2)) {
    ProtobufLogTestHelper.doWrite(new WriterOverAsyncWriter(writer), withTrailer, tableName,
      columnCount, recordCount, row, timestamp);
    try (ProtobufLogReader reader = (ProtobufLogReader) WALS.createReader(fs, path1)) {
      ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
        timestamp);
    }
    try (ProtobufLogReader reader = (ProtobufLogReader) WALS.createReader(fs, path2)) {
      ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
        timestamp);
    }
  }
}
 
Example #29
Source File: TestHFile.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void readNumMetablocks(Reader reader, int n) throws IOException {
  for (int i = 0; i < n; i++) {
    ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader();
    ByteBuffer expected =
      ByteBuffer.wrap(Bytes.toBytes("something to test" + i));
    assertEquals(
        "failed to match metadata",
        Bytes.toStringBinary(expected),
        Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(),
            actual.capacity()));
  }
}
 
Example #30
Source File: TestReversibleScanners.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static byte[][] makeN(byte[] base, int n) {
  byte[][] ret = new byte[n][];
  for (int i = 0; i < n; i++) {
    ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
  }
  return ret;
}