Java Code Examples for org.apache.hadoop.hbase.util.Bytes

The following examples show how to use org.apache.hadoop.hbase.util.Bytes. These examples are extracted from open source projects.
Example 1
Project: phoenix   File: TestTableRowkeyPair.java    License: Apache License 2.0 6 votes vote down vote up
private void testsRowsKeys(String aTable, String akey, String bTable, String bkey, int expectedSignum) throws IOException {
    
    final ImmutableBytesWritable arowkey = new ImmutableBytesWritable(Bytes.toBytes(akey));
    TableRowkeyPair pair1 = new TableRowkeyPair(aTable, arowkey);
    
    ImmutableBytesWritable browkey = new ImmutableBytesWritable(Bytes.toBytes(bkey));
    TableRowkeyPair pair2 = new TableRowkeyPair(bTable, browkey);
    
    TableRowkeyPair.Comparator comparator = new TableRowkeyPair.Comparator();
    try( ByteArrayOutputStream baosA = new ByteArrayOutputStream();
         ByteArrayOutputStream baosB = new ByteArrayOutputStream()) {
        
        pair1.write(new DataOutputStream(baosA));
        pair2.write(new DataOutputStream(baosB));
        Assert.assertEquals(expectedSignum , signum(pair1.compareTo(pair2)));
        Assert.assertEquals(expectedSignum , signum(comparator.compare(baosA.toByteArray(), 0, baosA.size(), baosB.toByteArray(), 0, baosB.size())));
        Assert.assertEquals(expectedSignum, -signum(comparator.compare(baosB.toByteArray(), 0, baosB.size(), baosA.toByteArray(), 0, baosA.size())));
    }

}
 
Example 2
Project: hbase-indexer   File: AbstractPrefixMatchingExtractor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Collection<byte[]> extract(Result result) {
    List<byte[]> values = Lists.newArrayList();

    NavigableMap<byte[], byte[]> qualifiersToValues = result.getFamilyMap(columnFamily);
    if (qualifiersToValues != null) {
        for (byte[] qualifier : qualifiersToValues.navigableKeySet().tailSet(prefix)) {
            if (Bytes.startsWith(qualifier, prefix)) {
                values.add(extractInternal(qualifier, qualifiersToValues.get(qualifier)));
            } else {
                break;
            }
        }
    }
    return values;
}
 
Example 3
/**
 * Test that we don't cover other columns when we have a delete column.
 */
@Test
public void testDeleteColumnCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue d = createKvForType(Type.DeleteColumn, 12);
  byte[] qual2 = Bytes.add(qualifier, Bytes.toBytes("-other"));
  KeyValue put =
      new KeyValue(row, family, qual2, 0, qual2.length, 11, Type.Put, value, 0,
          value.length);

  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // different column put should still be visible
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // set a delete family, but in the past
  d = createKvForType(Type.DeleteFamily, 10);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // add back in the original delete column
  d = createKvForType(Type.DeleteColumn, 11);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // onto a different family, so that must be visible too
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));
}
 
Example 4
Project: phoenix   File: SpillMap.java    License: Apache License 2.0 6 votes vote down vote up
private boolean canFit(byte[] curValue, byte[] newValue) {
    if (thresholdBytes < newValue.length) {
        // TODO resize page size if single element is too big,
        // Can this ever happen?
        throw new RuntimeException("page size too small to store a single KV element");
    }

    int resultSize = newValue.length + Bytes.SIZEOF_INT;
    if (curValue != null) {
        // Key existed before
        // Ensure to compensate for potential larger byte[] for agg
        resultSize = Math.max(0, resultSize - (curValue.length + Bytes.SIZEOF_INT));
    }

    if ((thresholdBytes - totalResultSize) <= (resultSize)) {
        // KV does not fit
        return false;
    }
    // KV fits
    return true;
}
 
Example 5
Project: phoenix   File: MultiEncodedCQKeyValueComparisonFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void write(DataOutput output) throws IOException {
    try {
        WritableUtils.writeVInt(output, minQualifier);
        WritableUtils.writeVInt(output, maxQualifier);
        WritableUtils.writeVInt(output, whereExpressionMinQualifier);
        WritableUtils.writeVInt(output, whereExpressionMaxQualifier);
        WritableUtils.writeVInt(output, encodingScheme.ordinal());
        super.write(output);
        output.writeBoolean(allCFs);
        if (!allCFs) {
            Bytes.writeByteArray(output, essentialCF);
        }
    } catch (DoNotRetryIOException e) {
        throw e;
    } catch (Throwable t) { // Catches incompatibilities during reading/writing and doesn't retry
        ServerUtil.throwIOException("MultiEncodedCQKeyValueComparisonFilter failed during writing", t);
    }
}
 
Example 6
Project: hbase   File: HelloHBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Invokes Admin#createNamespace and Admin#createTable to create a namespace
 * with a table that has one column-family.
 *
 * @param admin Standard Admin object
 * @throws IOException If IO problem encountered
 */
static void createNamespaceAndTable(final Admin admin) throws IOException {

  if (!namespaceExists(admin, MY_NAMESPACE_NAME)) {
    System.out.println("Creating Namespace [" + MY_NAMESPACE_NAME + "].");

    admin.createNamespace(NamespaceDescriptor
            .create(MY_NAMESPACE_NAME).build());
  }
  if (!admin.tableExists(MY_TABLE_NAME)) {
    System.out.println("Creating Table [" + MY_TABLE_NAME.getNameAsString()
            + "], with one Column Family ["
            + Bytes.toString(MY_COLUMN_FAMILY_NAME) + "].");

    admin.createTable(new TableDescriptorBuilder.ModifyableTableDescriptor(MY_TABLE_NAME)
      .setColumnFamily(
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
          MY_COLUMN_FAMILY_NAME)));
  }
}
 
Example 7
Project: hbase   File: TestSequenceIdMonotonicallyIncreasing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSplit()
    throws IOException, InterruptedException, ExecutionException, TimeoutException {
  try (Table table = createTable(false)) {
    table.put(new Put(Bytes.toBytes(0)).addColumn(CF, CQ, Bytes.toBytes(0)));
    table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(0)));
  }
  UTIL.flush(NAME);
  HRegionServer rs = UTIL.getRSForFirstRegionInTable(NAME);
  RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
  UTIL.getAdmin().splitRegionAsync(region.getRegionName(), Bytes.toBytes(1)).get(1,
    TimeUnit.MINUTES);
  long maxSeqId = getMaxSeqId(rs, region);
  RegionLocator locator = UTIL.getConnection().getRegionLocator(NAME);
  HRegionLocation locA = locator.getRegionLocation(Bytes.toBytes(0), true);
  HRegionLocation locB = locator.getRegionLocation(Bytes.toBytes(1), true);
  assertEquals(maxSeqId + 1, locA.getSeqNum());
  assertEquals(maxSeqId + 1, locB.getSeqNum());
}
 
Example 8
Project: hbase-indexer   File: LoggingConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.setBoolean("hbase.replication", true);

    ZooKeeperItf zk = ZkUtil.connect("localhost", 20000);
    SepModel sepModel = new SepModelImpl(zk, conf);

    final String subscriptionName = "logger";

    if (!sepModel.hasSubscription(subscriptionName)) {
        sepModel.addSubscriptionSilent(subscriptionName);
    }

    PayloadExtractor payloadExtractor = new BasePayloadExtractor(Bytes.toBytes("sep-user-demo"), Bytes.toBytes("info"),
            Bytes.toBytes("payload"));

    SepConsumer sepConsumer = new SepConsumer(subscriptionName, 0, new EventLogger(), 1, "localhost", zk, conf,
            payloadExtractor);

    sepConsumer.start();
    System.out.println("Started");

    while (true) {
        Thread.sleep(Long.MAX_VALUE);
    }
}
 
Example 9
Project: hbase-tools   File: MC.java    License: Apache License 2.0 6 votes vote down vote up
private void filterWithRsAndLocality(Set<byte[]> targets, String table) throws IOException {
    long startTimestamp = System.currentTimeMillis();
    Util.printVerboseMessage(args, Util.getMethodName() + " - start");

    Map<byte[], HRegionInfo> regionMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    String regex = (String) args.valueOf(Args.OPTION_REGION_SERVER);
    for (Map.Entry<HRegionInfo, ServerName> entry : getRegionLocations(table).entrySet()) {
        String serverName = entry.getValue().getHostname() + "," + entry.getValue().getPort();
        if (serverName.matches(regex)) {
            regionMap.put(entry.getKey().getRegionName(), entry.getKey());
            byte[] regionName = entry.getKey().getRegionName();
            targets.add(regionName);
            regionTableMap.put(regionName, table);
            regionRSMap.put(regionName, serverName);
        }
    }

    filterWithDataLocality(targets, regionMap);

    Util.printVerboseMessage(args, Util.getMethodName() + " - end", startTimestamp);
}
 
Example 10
Project: hbase   File: ScannerModel.java    License: Apache License 2.0 6 votes vote down vote up
public ByteArrayComparableModel(
    ByteArrayComparable comparator) {
  String typeName = comparator.getClass().getSimpleName();
  ComparatorType type = ComparatorType.valueOf(typeName);
  this.type = typeName;
  switch (type) {
    case BinaryComparator:
    case BinaryPrefixComparator:
      this.value = Bytes.toString(Base64.getEncoder().encode(comparator.getValue()));
      break;
    case BitComparator:
      this.value = Bytes.toString(Base64.getEncoder().encode(comparator.getValue()));
      this.op = ((BitComparator)comparator).getOperator().toString();
      break;
    case NullComparator:
      break;
    case RegexStringComparator:
    case SubstringComparator:
      this.value = Bytes.toString(comparator.getValue());
      break;
    default:
      throw new RuntimeException("unhandled filter type: " + type);
  }
}
 
Example 11
Project: phoenix   File: DistinctValueWithCountServerAggregator.java    License: Apache License 2.0 6 votes vote down vote up
private int countMapHeapSize() {
    int size = 0;
    if (this.valueVsCount.size() > 0) {
        for (ImmutableBytesPtr key : this.valueVsCount.keySet()) {
            size += SizedUtil.MAP_ENTRY_SIZE + // entry
                    Bytes.SIZEOF_INT + // key size
                    key.getLength() + SizedUtil.ARRAY_SIZE; // value size
        }
    } else {
        // Initially when the getSize() is called, we dont have any entries in the map so as to
        // tell the exact heap need. Let us approximate the #entries
        SizedUtil.sizeOfMap(DEFAULT_ESTIMATED_DISTINCT_VALUES,
                SizedUtil.IMMUTABLE_BYTES_PTR_SIZE, Bytes.SIZEOF_INT);
    }
    return size;
}
 
Example 12
Project: phoenix   File: AlterTableIT.java    License: Apache License 2.0 6 votes vote down vote up
private void assertEncodedCQValue(String columnFamily, String columnName, String schemaName, String tableName, int expectedValue) throws Exception {
    String query = "SELECT " + COLUMN_QUALIFIER + " FROM \"SYSTEM\".CATALOG WHERE " + TABLE_SCHEM + " = ? AND " + TABLE_NAME
            + " = ? " + " AND " + COLUMN_FAMILY + " = ?" + " AND " + COLUMN_NAME  + " = ?" + " AND " + COLUMN_QUALIFIER  + " IS NOT NULL";
    try (Connection conn = DriverManager.getConnection(getUrl())) {
        PreparedStatement stmt = conn.prepareStatement(query);
        stmt.setString(1, schemaName);
        stmt.setString(2, tableName);
        stmt.setString(3, columnFamily);
        stmt.setString(4, columnName);
        ResultSet rs = stmt.executeQuery();
        assertTrue(rs.next());
        if (columnEncoded) {
            assertTrue(Bytes.equals(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS.encode(expectedValue), rs.getBytes(1)));
        } else {
            assertTrue(Bytes.equals(columnName.getBytes(), rs.getBytes(1)));
        }
        assertFalse(rs.next());
    }
}
 
Example 13
Project: hbase   File: TestAtomicOperation.java    License: Apache License 2.0 6 votes vote down vote up
private void assertICV(byte [] row,
                       byte [] familiy,
                       byte[] qualifier,
                       long amount,
                       boolean fast) throws IOException {
  // run a get and see?
  Get get = new Get(row);
  if (fast) get.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
  get.addColumn(familiy, qualifier);
  Result result = region.get(get);
  assertEquals(1, result.size());

  Cell kv = result.rawCells()[0];
  long r = Bytes.toLong(CellUtil.cloneValue(kv));
  assertEquals(amount, r);
}
 
Example 14
Project: hbase-operator-tools   File: HBCKMetaTableAccessor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns all regions in meta for the given table.
 * @param conn a valid, open connection.
 * @param table the table to list regions in meta.
 * @return a list of <code>RegionInfo</code> for all table regions present in meta.
 * @throws IOException on any issues related with scanning meta table
 */
public static List<RegionInfo> getTableRegions(final Connection conn, final TableName table)
    throws IOException {
  final MetaScanner<RegionInfo> scanner = new MetaScanner<>();
  final String startRow = Bytes.toString(table.getName()) + ",,";
  final String stopRow = Bytes.toString(table.getName()) + " ,,";
  return scanner.scanMeta(conn,
    scan -> {
      scan.withStartRow(Bytes.toBytes(startRow));
      scan.withStopRow(Bytes.toBytes(stopRow));
    },
    r -> {
      Cell cell = r.getColumnLatestCell(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
      if(cell != null) {
        RegionInfo info = RegionInfo
          .parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
        return info;
      }
      return null;
    });
}
 
Example 15
Project: phoenix   File: TenantCacheTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExpiresButStaysInPersistentAfterTimeout() throws Exception {
    int maxServerCacheTimeToLive = 100;
    int maxServerCachePersistenceTimeToLive = 1000;
    long maxBytes = 1000;
    GlobalMemoryManager memoryManager = new GlobalMemoryManager(maxBytes);
    ManualTicker ticker = new ManualTicker();
    TenantCacheImpl cache = new TenantCacheImpl(memoryManager, maxServerCacheTimeToLive, maxServerCachePersistenceTimeToLive, ticker);
    ImmutableBytesPtr cacheId1 = new ImmutableBytesPtr(Bytes.toBytes(1L));
    ImmutableBytesWritable cachePtr = new ImmutableBytesWritable(Bytes.toBytes("a"));
    cache.addServerCache(cacheId1, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory, true, true, MetaDataProtocol.PHOENIX_VERSION);
    assertEquals(maxBytes-1, memoryManager.getAvailableMemory());
    assertNotNull(cache.getServerCache(cacheId1));

    // Expire it from live cache but not persistent cache
    ticker.time += (maxServerCacheTimeToLive + 1) * 1000000;
    cache.cleanUp();
    assertEquals(maxBytes-1, memoryManager.getAvailableMemory());
    assertNotNull(cache.getServerCache(cacheId1));

    // Expire it from persistent cache as well
    ticker.time += (maxServerCachePersistenceTimeToLive + 1) * 1000000;
    cache.cleanUp();
    assertEquals(maxBytes, memoryManager.getAvailableMemory());
    assertNull(cache.getServerCache(cacheId1));
}
 
Example 16
Project: hbase   File: TestHFileOutputFormat2.java    License: Apache License 2.0 6 votes vote down vote up
private void setupMockColumnFamiliesForCompression(Table table,
    Map<String, Compression.Algorithm> familyToCompression) throws IOException {

  TableDescriptorBuilder mockTableDescriptor =
    TableDescriptorBuilder.newBuilder(TABLE_NAMES[0]);
  for (Entry<String, Compression.Algorithm> entry : familyToCompression.entrySet()) {
    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(entry.getKey()))
      .setMaxVersions(1)
      .setCompressionType(entry.getValue())
      .setBlockCacheEnabled(false)
      .setTimeToLive(0)
      .build();

    mockTableDescriptor.setColumnFamily(columnFamilyDescriptor);
  }
  Mockito.doReturn(mockTableDescriptor.build()).when(table).getDescriptor();
}
 
Example 17
Project: hbase   File: RowResourceBase.java    License: Apache License 2.0 5 votes vote down vote up
protected static void checkValueXML(String table, String row, String column,
    String value) throws IOException, JAXBException {
  Response response = getValueXML(table, row, column);
  assertEquals(200, response.getCode());
  assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
  CellSetModel cellSet = (CellSetModel)
    xmlUnmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
  RowModel rowModel = cellSet.getRows().get(0);
  CellModel cell = rowModel.getCells().get(0);
  assertEquals(Bytes.toString(cell.getColumn()), column);
  assertEquals(Bytes.toString(cell.getValue()), value);
}
 
Example 18
Project: hbase   File: CheckAndMutate.java    License: Apache License 2.0 5 votes vote down vote up
private void preCheck(Row action) {
  Preconditions.checkNotNull(action, "action (Put/Delete/RowMutations) is null");
  if (!Bytes.equals(row, action.getRow())) {
    throw new IllegalArgumentException("The row of the action (Put/Delete/RowMutations) <" +
      Bytes.toStringBinary(action.getRow()) + "> doesn't match the original one <" +
      Bytes.toStringBinary(this.row) + ">");
  }
  Preconditions.checkState(op != null || filter != null, "condition is null. You need to"
    + " specify the condition by calling ifNotExists/ifEquals/ifMatches before building a"
    + " CheckAndMutate object");
}
 
Example 19
Project: hbase   File: TestGroupingTableMapper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test GroupingTableMapper class
 */
@Test
public void testGroupingTableMapper() throws Exception {

  GroupingTableMapper mapper = new GroupingTableMapper();
  Configuration configuration = new Configuration();
  configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm");
  mapper.setConf(configuration);

  Result result = mock(Result.class);
  @SuppressWarnings("unchecked")
  Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context =
      mock(Mapper.Context.class);
  context.write(any(), any());
  List<Cell> keyValue = new ArrayList<>();
  byte[] row = {};
  keyValue.add(new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes
      .toBytes("value1")));
  keyValue.add(new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes
      .toBytes("value2")));
  when(result.listCells()).thenReturn(keyValue);
  mapper.map(null, result, context);
  // template data
  byte[][] data = { Bytes.toBytes("value1"), Bytes.toBytes("value2") };
  ImmutableBytesWritable ibw = mapper.createGroupKey(data);
  verify(context).write(ibw, result);
}
 
Example 20
Project: phoenix-omid   File: TestAutoFlush.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut = 10_000)
public void testReadWithSeveralUncommitted(ITestContext context) throws Exception {

    byte[] family = Bytes.toBytes(TEST_FAMILY);
    byte[] row = Bytes.toBytes("row");
    byte[] col = Bytes.toBytes("col1");
    byte[] data = Bytes.toBytes("data");
    TransactionManager tm = newTransactionManager(context);
    TTable table = new TTable(connection, TEST_TABLE);

    // Turn off autoflush
    table.setAutoFlush(false);

    Transaction t = tm.begin();
    Put put = new Put(row);
    put.addColumn(family, col, data);
    table.put(t, put);

    // Data shouldn't be in DB yet
    Get get = new Get(row);
    Result result = table.getHTable().get(get);
    assertEquals(result.size(), 0, "Writes are already in DB");

    //data should be readable within same transaction
    result = table.get(t,get);
    assertEquals(result.size(), 1, "Writes should be read by same transaction");

    tm.commit(t);

    // After commit, both the cell and shadow cell should be there.
    // That's why we check for two elements in the test assertion
    result = table.getHTable().get(get);
    assertEquals(result.size(), 2, "Writes were not flushed to DB");
}
 
Example 21
Project: hbase   File: PlainTextMessageBodyProducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeTo(Object object, Class<?> type, Type genericType,
    Annotation[] annotations, MediaType mediaType,
    MultivaluedMap<String, Object> httpHeaders, OutputStream outStream)
    throws IOException, WebApplicationException {
  outStream.write(Bytes.toBytes(object.toString()));
}
 
Example 22
Project: hbase   File: ScanModifyingObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void start(
    @SuppressWarnings("rawtypes") CoprocessorEnvironment env) throws IOException {
  RegionCoprocessorEnvironment renv = (RegionCoprocessorEnvironment) env;
  FAMILY_TO_ADD = Bytes.toBytes(renv.getConfiguration().get(FAMILY_TO_ADD_KEY));
  QUALIFIER_TO_ADD = Bytes.toBytes(renv.getConfiguration().get(QUALIFIER_TO_ADD_KEY));
}
 
Example 23
Project: phoenix-tephra   File: PruneUpperBoundWriter.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("WeakerAccess")
public PruneUpperBoundWriter(TableName tableName, DataJanitorState dataJanitorState, long pruneFlushInterval) {
  this.tableName = tableName;
  this.dataJanitorState = dataJanitorState;
  this.pruneFlushInterval = pruneFlushInterval;
  this.pruneEntries = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
  this.emptyRegions = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
}
 
Example 24
Project: HBase-ToHDFS   File: ExportHBaseTableToAvro.java    License: Apache License 2.0 5 votes vote down vote up
private void putValue(Record record, String col, byte[] value) {
  if (schema.getField(col).schema().getType().equals(Type.STRING)) {
    record.put(col, Bytes.toString(value));
  } else if (schema.getField(col).schema().getType().equals(Type.INT)) {
    record.put(col, Bytes.toInt(value));
  } else if (schema.getField(col).schema().getType().equals(Type.LONG)) {
    record.put(col, Bytes.toLong(value));
  } else {
    throw new RuntimeException("Unknown datatype: " + schema.getField(col).schema().getType());
  }
}
 
Example 25
Project: phoenix   File: PUnsignedFloat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public float decodeFloat(byte[] b, int o, SortOrder sortOrder) {
  Preconditions.checkNotNull(sortOrder);
  checkForSufficientLength(b, o, Bytes.SIZEOF_FLOAT);
  if (sortOrder == SortOrder.DESC) {
    b = SortOrder.invert(b, o, new byte[Bytes.SIZEOF_FLOAT], 0, Bytes.SIZEOF_FLOAT);
    o = 0;
  }
  float v = Bytes.toFloat(b, o);
  if (v < 0) {
    throw newIllegalDataException();
  }
  return v;
}
 
Example 26
Project: metron   File: StellarTransformationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntermediateValues() throws Exception {

  SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig));
  FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
  JSONObject input = new JSONObject(new HashMap<String, Object>() {{
  }});
  handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
  int expected = 3;
  assertEquals(expected, input.get("final_value"));
  assertFalse(input.containsKey("value1"));
  assertFalse(input.containsKey("value2"));
}
 
Example 27
Project: spliceengine   File: BackupUtils.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
/**
 * If the end key from the request equals to the region start key, the request is meant for the previous region.
 * Ignore shuch requests.
 * @param request
 * @param region
 * @return
 */
public static boolean regionKeysMatch(SpliceMessage.PrepareBackupRequest request, HRegion region) {
    byte[] requestStartKey = request.hasStartKey() ? request.getStartKey().toByteArray() : new byte[0];
    byte[] requestEndKey = request.hasEndKey() ? request.getEndKey().toByteArray() : new byte[0];

    byte[] regionStartKey = region.getRegionInfo().getStartKey() != null? region.getRegionInfo().getStartKey() : new byte[0];
    byte[] regionEndKey = region.getRegionInfo().getEndKey() != null ? region.getRegionInfo().getEndKey() : new byte[0];

    return Bytes.compareTo(requestStartKey, regionStartKey) ==0 &&
            Bytes.compareTo(requestEndKey, regionEndKey) == 0;
}
 
Example 28
Project: hbase   File: TableSplit.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean equals(Object o) {
  if (o == null || !(o instanceof TableSplit)) {
    return false;
  }
  TableSplit other = (TableSplit)o;
  return m_tableName.equals(other.m_tableName) &&
    Bytes.equals(m_startRow, other.m_startRow) &&
    Bytes.equals(m_endRow, other.m_endRow) &&
    m_regionLocation.equals(other.m_regionLocation);
}
 
Example 29
Project: presto-hbase-connector   File: HBaseRecordCursor.java    License: Apache License 2.0 5 votes vote down vote up
void setRowKeyValue2FieldsAry(Result record, int fieldIndex) {
    // Handle the value of rowKey
    // Check out whether columns to be queried contain rowKey field
    if (fieldIndexMap.containsKey(this.rowKeyColName.hashCode())) {
        if (record.rawCells() != null && record.rawCells().length > 0) {
            Cell cell = record.rawCells()[0];
            HBaseColumnHandle rowKeyHandle = fieldIndexMap.get(rowKeyColName.hashCode());
            String rowKeyValue = Bytes.toString(
                    arrayCopy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
            fields[fieldIndex] = rowKeyValue;
            ordinalPositionAndFieldsIndexMap.put(rowKeyHandle.getOrdinalPosition(), fieldIndex);
        }
    }
}
 
Example 30
Project: phoenix   File: UpgradeUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Synchronize certain properties across column families of view index tables for a given base table
 * @param cqs CQS object to get table descriptor from PTable
 * @param baseTable base table
 * @param defaultColFam column family to be used for synchronizing properties
 * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor
 * @param tableDescsToSync set of modified table descriptors
 */
private static void syncViewIndexTable(ConnectionQueryServices cqs, PTable baseTable, ColumnFamilyDescriptor defaultColFam,
        Map<String, Object> syncedProps, Set<TableDescriptor> tableDescsToSync) throws SQLException {
    String viewIndexName = MetaDataUtil.getViewIndexPhysicalName(baseTable.getPhysicalName().getString());
    if (!Strings.isNullOrEmpty(viewIndexName)) {
        try {
            addTableDescIfPropsChanged(cqs.getTableDescriptor(Bytes.toBytes(viewIndexName)),
                    defaultColFam, syncedProps, tableDescsToSync);
        } catch (TableNotFoundException ignore) {
            // Ignore since this means that a view index table does not exist for this table
        }
    }
}
 
Example 31
Project: hbase   File: TestStoreScannerClosure.java    License: Apache License 2.0 5 votes vote down vote up
NavigableSet<byte[]> getCols(String... strCols) {
  NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  for (String col : strCols) {
    byte[] bytes = Bytes.toBytes(col);
    cols.add(bytes);
  }
  return cols;
}
 
Example 32
Project: searchanalytics-bigdata   File: HbaseServiceImpl.java    License: MIT License 5 votes vote down vote up
@Override
public int findTotalRecordsForValidCustomers() {
	int totalCount = 0;
	List<String> rows = hbaseTemplate.find("searchclicks", new String(
			HbaseJsonEventSerializer.COLUMFAMILY_CLIENT_BYTES),
			new RowMapper<String>() {
				@Override
				public String mapRow(Result result, int rowNum)
						throws Exception {
					String customerid = new String(
							result.getValue(
									HbaseJsonEventSerializer.COLUMFAMILY_CLIENT_BYTES,
									Bytes.toBytes("customerid")));
					return customerid;
				}
			});
	for (String row : rows) {
		LOG.debug(
				"Table count findTotalRecordsForValidCustomers returned row is : {}",
				row);
		if (row != null) {
			totalCount++;
		}
	}
	LOG.debug(
			"findTotalRecordsForValidCustomers searchclicks table count is: {}",
			totalCount);
	return totalCount;
}
 
Example 33
Project: hbase   File: TestHFile.java    License: Apache License 2.0 5 votes vote down vote up
private void readNumMetablocks(Reader reader, int n) throws IOException {
  for (int i = 0; i < n; i++) {
    ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader();
    ByteBuffer expected =
      ByteBuffer.wrap(Bytes.toBytes("something to test" + i));
    assertEquals(
        "failed to match metadata",
        Bytes.toStringBinary(expected),
        Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(),
            actual.capacity()));
  }
}
 
Example 34
Project: hbase   File: TestCombinedAsyncWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void doTest(boolean withTrailer) throws IOException {
  int columnCount = 5;
  int recordCount = 5;
  TableName tableName = TableName.valueOf("tablename");
  byte[] row = Bytes.toBytes("row");
  long timestamp = System.currentTimeMillis();
  Path path1 = getPath(1);
  Path path2 = getPath(2);
  FileSystem fs = UTIL.getTestFileSystem();
  Configuration conf = UTIL.getConfiguration();
  try (
    AsyncWriter writer1 = AsyncFSWALProvider.createAsyncWriter(conf, fs, path1, false,
      EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
    AsyncWriter writer2 = AsyncFSWALProvider.createAsyncWriter(conf, fs, path2, false,
      EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
    CombinedAsyncWriter writer = CombinedAsyncWriter.create(writer1, writer2)) {
    ProtobufLogTestHelper.doWrite(new WriterOverAsyncWriter(writer), withTrailer, tableName,
      columnCount, recordCount, row, timestamp);
    try (ProtobufLogReader reader = (ProtobufLogReader) WALS.createReader(fs, path1)) {
      ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
        timestamp);
    }
    try (ProtobufLogReader reader = (ProtobufLogReader) WALS.createReader(fs, path2)) {
      ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
        timestamp);
    }
  }
}
 
Example 35
Project: PoseidonX   File: HBaseUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 获得一个字符串的字节数组
 * @param value
 * @return
 */
public static byte[] getBytes(String value) {
	if (value == null) {
		return null;
	}
	return Bytes.toBytes(value);
}
 
Example 36
Project: hbase   File: TestSimpleTotalOrderPartitioner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSplit() throws Exception {
  String start = "a";
  String end = "{";
  SimpleTotalOrderPartitioner<byte []> p = new SimpleTotalOrderPartitioner<>();

  this.conf.set(SimpleTotalOrderPartitioner.START, start);
  this.conf.set(SimpleTotalOrderPartitioner.END, end);
  p.setConf(this.conf);
  ImmutableBytesWritable c = new ImmutableBytesWritable(Bytes.toBytes("c"));
  // If one reduce, partition should be 0.
  int partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 1);
  assertEquals(0, partition);
  // If two reduces, partition should be 0.
  partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(0, partition);
  // Divide in 3.
  partition = p.getPartition(c, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(0, partition);
  ImmutableBytesWritable q = new ImmutableBytesWritable(Bytes.toBytes("q"));
  partition = p.getPartition(q, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(1, partition);
  partition = p.getPartition(q, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(2, partition);
  // What about end and start keys.
  ImmutableBytesWritable startBytes =
    new ImmutableBytesWritable(Bytes.toBytes(start));
  partition = p.getPartition(startBytes, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(0, partition);
  partition = p.getPartition(startBytes, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(0, partition);
  ImmutableBytesWritable endBytes =
    new ImmutableBytesWritable(Bytes.toBytes("z"));
  partition = p.getPartition(endBytes, HConstants.EMPTY_BYTE_ARRAY, 2);
  assertEquals(1, partition);
  partition = p.getPartition(endBytes, HConstants.EMPTY_BYTE_ARRAY, 3);
  assertEquals(2, partition);
}
 
Example 37
Project: hbase   File: TestReversibleScanners.java    License: Apache License 2.0 5 votes vote down vote up
private static byte[][] makeN(byte[] base, int n) {
  byte[][] ret = new byte[n][];
  for (int i = 0; i < n; i++) {
    ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
  }
  return ret;
}
 
Example 38
Project: hbase   File: TestVisibilityLabelsWithDeletes.java    License: Apache License 2.0 5 votes vote down vote up
private void scanAll(Result[] next) throws IOException {
  CellScanner cellScanner = next[0].cellScanner();
  cellScanner.advance();
  Cell current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(127L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(126L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(125L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(124L, current.getTimestamp());
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row1, 0, row1.length));
  assertEquals(123L, current.getTimestamp());
  cellScanner = next[1].cellScanner();
  cellScanner.advance();
  current = cellScanner.current();
  assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
    row2, 0, row2.length));
}
 
Example 39
Project: hbase   File: TestMobFileCache.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  UTIL = new HBaseTestingUtility();
  conf = UTIL.getConfiguration();
  conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(UTIL.createTableDescriptor(
      TableName.valueOf("testMobFileCache"), ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS,
      3, HConstants.FOREVER, ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY1))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY2))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY3))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  mobFileCache = new MobFileCache(conf);
  region = HBaseTestingUtility
    .createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf, tableDescriptor, mobFileCache);
}
 
Example 40
Project: canal-1.1.3   File: PhTypeUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static int encodeShort(short v, byte[] b, int o) {
    checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT);
    b[o + 0] = (byte) ((v >> 8) ^ 0x80); // Flip sign bit so that Short is
                                         // binary comparable
    b[o + 1] = (byte) v;
    return Bytes.SIZEOF_SHORT;
}