Java Code Examples for org.apache.hadoop.hbase.util.Bytes#toBytes()

The following examples show how to use org.apache.hadoop.hbase.util.Bytes#toBytes() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: SkyEye   File: HbaseStore.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * 数据表trace, 保存一个跟踪链条的所有span信息
 * rowkey: traceId
 * columnFamily: span
 * qualifier: [spanId+c, spanId+s ...](有N个span就有N*2个, c/s表示是client还是server采集到的)
 * value: span json value
 * @param span
 * @param spanJson
 * @param annotationMap
 * @return
 */
@Override
public Put storeSpan(Span span, String spanJson, Map<String, Annotation> annotationMap) {
    String traceId = span.getTraceId();
    String spanId = span.getId();
    if (annotationMap.containsKey(AnnotationType.CS.symbol())) {
        // 如果是client
        spanId += NodeProperty.C.symbol();
    } else {
        // 如果是server
        spanId += NodeProperty.S.symbol();
    }
    Put put = new Put(Bytes.toBytes(traceId));
    put.addColumn(Bytes.toBytes(Constants.TABLE_TRACE_COLUMN_FAMILY), Bytes.toBytes(spanId), Bytes.toBytes(spanJson));

    return put;
}
 
Example 2
/**
 * Run MR job to test autobalance for setting number of mappers for TIF This does not run real MR
 * job
 */
protected void testAutobalanceNumOfSplit() throws IOException {
  // set up splits for testing
  List<InputSplit> splits = new ArrayList<>(5);
  int[] regionLen = { 10, 20, 20, 40, 60 };
  for (int i = 0; i < 5; i++) {
    InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i),
      Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576);
    splits.add(split);
  }
  TableInputFormat tif = new TableInputFormat();
  List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824);

  assertEquals("Saw the wrong number of splits", 5, res.size());
  TableSplit ts1 = (TableSplit) res.get(0);
  assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow()));
  TableSplit ts2 = (TableSplit) res.get(1);
  assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength());
  TableSplit ts3 = (TableSplit) res.get(2);
  assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow()));
  TableSplit ts4 = (TableSplit) res.get(4);
  assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow()));
}
 
Example 3
Source Project: hbase   File: KeyValueTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static KeyValue create(
    String row,
    String family,
    String qualifier,
    long timestamp,
    KeyValue.Type type,
    String value)
{
    return new KeyValue(
        Bytes.toBytes(row),
        Bytes.toBytes(family),
        Bytes.toBytes(qualifier),
        timestamp,
        type,
        Bytes.toBytes(value)
    );
}
 
Example 4
public void addNodes(ArrayList<NodePojo> nodes) throws IOException {
  Table nodeTable = connection.getTable(HBaseConst.NODE_LIST_TABLE);

  ArrayList<Put> putList = new ArrayList<Put>();

  for (NodePojo node: nodes) {
    Put put = new Put(Bytes.toBytes(node.getIpAddress()));
    put.addColumn(HBaseConst.BASE_COLUMN_FAMILY, Bytes.toBytes("Nm"), Bytes.toBytes(node.getName()));
    put.addColumn(HBaseConst.BASE_COLUMN_FAMILY, Bytes.toBytes("Zip"), Bytes.toBytes(node.getZipCode()));
    put.addColumn(HBaseConst.BASE_COLUMN_FAMILY, Bytes.toBytes("Grp"), Bytes.toBytes(node.getGroup()));
    putList.add(put);
  }
  nodeTable.put(putList);

  nodeTable.close();
}
 
Example 5
Source Project: hbase   File: TestRegionIncrement.java    License: Apache License 2.0 5 votes vote down vote up
CrossRowCellIncrementer(final int i, final int count, final HRegion region, final int range) {
  super("" + i);
  setDaemon(true);
  this.count = count;
  this.region = region;
  this.increments = new Increment[range];
  for (int ii = 0; ii < range; ii++) {
    this.increments[ii] = new Increment(Bytes.toBytes(i));
    this.increments[ii].addColumn(INCREMENT_BYTES, INCREMENT_BYTES, 1);
  }
}
 
Example 6
Source Project: hbase   File: TestStoreScannerClosure.java    License: Apache License 2.0 5 votes vote down vote up
NavigableSet<byte[]> getCols(String... strCols) {
  NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  for (String col : strCols) {
    byte[] bytes = Bytes.toBytes(col);
    cols.add(bytes);
  }
  return cols;
}
 
Example 7
@Test
public void testSameTypeOfMutationWithSameValueButDifferentTimeStamp() throws Exception {
    SetupInfo info = setup(TABLE_NAME,
            INDEX_NAME,
            "ROW_KEY VARCHAR, C1 VARCHAR, C2 VARCHAR",
            "C1",
            "ROW_KEY",
            "");

    Put dataPut = new Put(Bytes.toBytes(ROW_KEY));
    addCellToPutMutation(dataPut,
            info.indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
            Bytes.toBytes("C2"),
            1,
            Bytes.toBytes("v2"));
    addEmptyColumnToDataPutMutation(dataPut, info.pDataTable, 1);
    addCellToPutMutation(dataPut,
            info.indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
            Bytes.toBytes("C2"),
            1,
            Bytes.toBytes("v3"));
    addEmptyColumnToDataPutMutation(dataPut, info.pDataTable, 2);

    List<Mutation> actualIndexMutations = IndexRebuildRegionScanner.prepareIndexMutationsForRebuild(info.indexMaintainer,
            dataPut,
            null);

    byte[] idxKeyBytes = generateIndexRowKey(null);

    // idxPut1 and idxPut2 have same value but different time stamp
    Put idxPut1 = new Put(idxKeyBytes);
    addEmptyColumnToIndexPutMutation(idxPut1, info.indexMaintainer, 1);

    Put idxPut2 = new Put(idxKeyBytes);
    addEmptyColumnToIndexPutMutation(idxPut2, info.indexMaintainer, 2);

    assertEqualMutationList(Arrays.asList((Mutation)idxPut1, (Mutation)idxPut2), actualIndexMutations);
}
 
Example 8
@Override
	protected void map(LongWritable key, Text value, Context context)
			throws IOException, InterruptedException {

		try {
			String lineString = value.toString();
			String[] arr = lineString.split("\t", -1);
			if (arr.length == 2) {
				String midTs = arr[0];
				String cfq = arr[1];
				String[] keys = midTs.split(Const.FIELD_COMMON_SEPARATOR, -1);
				if (keys.length == 2) {
					rowkey = Bytes.toBytes(keys[0]);
					ts = Long.parseLong(keys[1]);
				}

				String[] vals = cfq.split(Const.FIELD_COMMON_SEPARATOR, -1);
				if (vals.length == 3) {
					family = Bytes.toBytes(vals[0]);
					qualifier = Bytes.toBytes(vals[1]);
					cellValue = Bytes.toBytes(vals[2]);

				}

				Put put = new Put(rowkey, ts);
				put.add(family, qualifier, cellValue);

				context.write(new ImmutableBytesWritable(rowkey), put);
//				table.incrementColumnValue(rowkey, family,
//						Const.COLUMN_RK_COUNTER_BYTE, 1L);
				long cur = table.incrementColumnValue(rowkey, family,
						Const.COLUMN_RK_COUNTER_BYTE, 1L);
				System.out.println(cur);
			
			}
		} catch (Exception e) {
			e.printStackTrace();
		}

	}
 
Example 9
Source Project: hbase   File: TestRegionInfo.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("SelfComparison")
@Test
public void testComparator() {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo older = RegionInfoBuilder.newBuilder(tableName).setRegionId(0).build();
  RegionInfo newer = RegionInfoBuilder.newBuilder(tableName).setRegionId(1).build();
  assertTrue(older.compareTo(newer) < 0);
  assertTrue(newer.compareTo(older) > 0);
  assertEquals(0, older.compareTo(older));
  assertEquals(0, newer.compareTo(newer));

  RegionInfo a = RegionInfoBuilder.newBuilder(TableName.valueOf("a")).build();
  RegionInfo b = RegionInfoBuilder.newBuilder(TableName.valueOf("b")).build();
  assertNotEquals(0, a.compareTo(b));
  TableName t = TableName.valueOf("t");
  byte [] midway = Bytes.toBytes("midway");
  a = RegionInfoBuilder.newBuilder(t).setEndKey(midway).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(midway).build();
  assertTrue(a.compareTo(b) < 0);
  assertTrue(b.compareTo(a) > 0);
  assertEquals(a, a);
  assertEquals(0, a.compareTo(a));
  a = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("a"))
    .setEndKey(Bytes.toBytes("d")).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("e"))
    .setEndKey(Bytes.toBytes("g")).build();
  assertTrue(a.compareTo(b) < 0);
  a = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("aaaa"))
    .setEndKey(Bytes.toBytes("dddd")).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("e"))
    .setEndKey(Bytes.toBytes("g")).build();
  assertTrue(a.compareTo(b) < 0);
  a = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("aaaa"))
    .setEndKey(Bytes.toBytes("dddd")).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("aaaa"))
    .setEndKey(Bytes.toBytes("eeee")).build();
  assertTrue(a.compareTo(b) < 0);

}
 
Example 10
@Test
public void testTruncateWithPreserveAfterSplit() throws Exception {
  String[] families = new String[] { "f1", "f2" };
  byte[][] splitKeys =
    new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
  TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
    tableName, splitKeys, families);
  splitAndTruncate(tableName, regions, 1);
}
 
Example 11
@Override
public void deleteApplicationName(String applicationName) {
    Objects.requireNonNull(applicationName, "applicationName");

    byte[] rowKey = Bytes.toBytes(applicationName);
    Delete delete = new Delete(rowKey);

    TableName applicationIndexTableName = descriptor.getTableName();
    hbaseOperations2.delete(applicationIndexTableName, delete);
}
 
Example 12
Source Project: hbase   File: RandomKeyValueUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static byte[] randomRowOrQualifier(Random rand) {
  StringBuilder field = new StringBuilder();
  int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
      + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
      - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
  for (int i = 0; i < fieldLen; ++i)
    field.append(randomReadableChar(rand));
  return Bytes.toBytes(field.toString());
}
 
Example 13
Source Project: phoenix   File: CsvToKeyValueMapper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void setup(Context context) throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    String jdbcUrl = getJdbcUrl(conf);

    // pass client configuration into driver
    Properties clientInfos = new Properties();
    Iterator<Entry<String, String>> iterator = conf.iterator();
    while(iterator.hasNext()) {
        Entry<String,String> entry = iterator.next();
        clientInfos.setProperty(entry.getKey(), entry.getValue());
    }
    
    // This statement also ensures that the driver class is loaded
    LOG.info("Connection with driver {} with url {}", PhoenixDriver.class.getName(), jdbcUrl);

    try {
        conn = (PhoenixConnection) DriverManager.getConnection(jdbcUrl, clientInfos);
    } catch (SQLException e) {
        throw new RuntimeException(e);
    }

    upsertListener = new MapperUpsertListener(
            context, conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true));
    csvUpsertExecutor = buildUpsertExecutor(conf);
    csvLineParser = new CsvLineParser(conf.get(FIELD_DELIMITER_CONFKEY).charAt(0), conf.get(QUOTE_CHAR_CONFKEY).charAt(0),
            conf.get(ESCAPE_CHAR_CONFKEY).charAt(0));

    preUpdateProcessor = loadPreUpsertProcessor(conf);
    if(!conf.get(CsvToKeyValueMapper.INDEX_TABLE_NAME_CONFKEY, "").isEmpty()){
    	tableName = Bytes.toBytes(conf.get(CsvToKeyValueMapper.INDEX_TABLE_NAME_CONFKEY));
    } else {
    	tableName = Bytes.toBytes(conf.get(CsvToKeyValueMapper.TABLE_NAME_CONFKEY, ""));
    }
}
 
Example 14
Source Project: phoenix-omid   File: TestSnapshotFilter.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeOut = 60_000)
public void testScanSecondResult() throws Throwable {

    byte[] rowName1 = Bytes.toBytes("row1");
    byte[] famName1 = Bytes.toBytes(TEST_FAMILY);
    byte[] colName1 = Bytes.toBytes("col1");
    byte[] dataValue1 = Bytes.toBytes("testWrite-1");

    String TEST_TABLE = "testScanSecondResult";
    createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
    TTable tt = new TTable(connection, TEST_TABLE);

    Transaction tx1 = tm.begin();

    Put put1 = new Put(rowName1);
    put1.addColumn(famName1, colName1, dataValue1);
    tt.put(tx1, put1);

    tm.commit(tx1);

    Transaction tx2 = tm.begin();

    Put put2 = new Put(rowName1);
    put2.addColumn(famName1, colName1, dataValue1);
    tt.put(tx2, put2);

    Transaction tx3 = tm.begin();

    ResultScanner iterableRS = tt.getScanner(tx3, new Scan().setStartRow(rowName1).setStopRow(rowName1));
    Result result = iterableRS.next();
    long tsRow = result.rawCells()[0].getTimestamp();
    assertEquals(tsRow, tx1.getTransactionId(), "Reading differnt version");

    assertFalse(iterableRS.next() != null);

    tm.commit(tx3);

    tt.close();
}
 
Example 15
@Test
public void testLabelsWithAppend() throws Throwable {
  PrivilegedExceptionAction<VisibilityLabelsResponse> action =
      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
    @Override
    public VisibilityLabelsResponse run() throws Exception {
      try (Connection conn = ConnectionFactory.createConnection(conf)) {
        return VisibilityClient.setAuths(conn, new String[] { TOPSECRET },
            USER.getShortName());
      } catch (Throwable e) {
      }
      return null;
    }
  };
  SUPERUSER.runAs(action);
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = TEST_UTIL.createTable(tableName, fam)) {
    final byte[] row1 = Bytes.toBytes("row1");
    final byte[] val = Bytes.toBytes("a");
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Put put = new Put(row1);
          put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
          put.setCellVisibility(new CellVisibility(TOPSECRET));
          table.put(put);
        }
        return null;
      }
    };
    USER.runAs(actiona);
    actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Append append = new Append(row1);
          append.addColumn(fam, qual, Bytes.toBytes("b"));
          table.append(append);
        }
        return null;
      }
    };
    USER.runAs(actiona);
    actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Append append = new Append(row1);
          append.addColumn(fam, qual, Bytes.toBytes("c"));
          append.setCellVisibility(new CellVisibility(PUBLIC));
          table.append(append);
          Assert.fail("Testcase should fail with AccesDeniedException");
        } catch (Throwable t) {
          assertTrue(t.getMessage().contains("AccessDeniedException"));
        }
        return null;
      }
    };
    USER.runAs(actiona);
  }
}
 
Example 16
@Test
public void testNoInsertsWithIncrement() throws Exception {
  Increment i = new Increment(Bytes.toBytes("to_reject"));
  i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
  helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
}
 
Example 17
Source Project: hbase   File: AbstractTestWALReplay.java    License: Apache License 2.0 4 votes vote down vote up
/**
 *
 * @throws Exception
 */
@Test
public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
  final TableName tableName =
      TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
  byte[] family1 = Bytes.toBytes("cf1");
  byte[] family2 = Bytes.toBytes("cf2");
  byte[] qualifier = Bytes.toBytes("q");
  byte[] value = Bytes.toBytes("testV");
  byte[][] familys = { family1, family2 };
  TEST_UTIL.createTable(tableName, familys);
  Table htable = TEST_UTIL.getConnection().getTable(tableName);
  Put put = new Put(Bytes.toBytes("r1"));
  put.addColumn(family1, qualifier, value);
  htable.put(put);
  ResultScanner resultScanner = htable.getScanner(new Scan());
  int count = 0;
  while (resultScanner.next() != null) {
    count++;
  }
  resultScanner.close();
  assertEquals(1, count);

  MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
  List<HRegion> regions = hbaseCluster.getRegions(tableName);
  assertEquals(1, regions.size());

  // move region to another regionserver
  Region destRegion = regions.get(0);
  int originServerNum = hbaseCluster.getServerWith(destRegion.getRegionInfo().getRegionName());
  assertTrue("Please start more than 1 regionserver",
      hbaseCluster.getRegionServerThreads().size() > 1);
  int destServerNum = 0;
  while (destServerNum == originServerNum) {
    destServerNum++;
  }
  HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
  HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
  // move region to destination regionserver
  TEST_UTIL.moveRegionAndWait(destRegion.getRegionInfo(), destServer.getServerName());

  // delete the row
  Delete del = new Delete(Bytes.toBytes("r1"));
  htable.delete(del);
  resultScanner = htable.getScanner(new Scan());
  count = 0;
  while (resultScanner.next() != null) {
    count++;
  }
  resultScanner.close();
  assertEquals(0, count);

  // flush region and make major compaction
  HRegion region =
      (HRegion) destServer.getOnlineRegion(destRegion.getRegionInfo().getRegionName());
  region.flush(true);
  // wait to complete major compaction
  for (HStore store : region.getStores()) {
    store.triggerMajorCompaction();
  }
  region.compact(true);

  // move region to origin regionserver
  TEST_UTIL.moveRegionAndWait(destRegion.getRegionInfo(), originServer.getServerName());
  // abort the origin regionserver
  originServer.abort("testing");

  // see what we get
  Result result = htable.get(new Get(Bytes.toBytes("r1")));
  if (result != null) {
    assertTrue("Row is deleted, but we get" + result.toString(),
        (result == null) || result.isEmpty());
  }
  resultScanner.close();
}
 
Example 18
@Test
public void testSplitFailedCompactionAndSplit() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  // Create table then get the single region for our new table.
  byte[] cf = Bytes.toBytes("cf");
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
  admin.createTable(htd);

  for (int i = 0; cluster.getRegions(tableName).isEmpty() && i < 100; i++) {
    Thread.sleep(100);
  }
  assertEquals(1, cluster.getRegions(tableName).size());

  HRegion region = cluster.getRegions(tableName).get(0);
  HStore store = region.getStore(cf);
  int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
  HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);

  Table t = TESTING_UTIL.getConnection().getTable(tableName);
  // insert data
  insertData(tableName, admin, t);
  insertData(tableName, admin, t);

  int fileNum = store.getStorefiles().size();
  // 0, Compaction Request
  store.triggerMajorCompaction();
  Optional<CompactionContext> cc = store.requestCompaction();
  assertTrue(cc.isPresent());
  // 1, A timeout split
  // 1.1 close region
  assertEquals(2, region.close(false).get(cf).size());
  // 1.2 rollback and Region initialize again
  region.initialize();

  // 2, Run Compaction cc
  assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE));
  assertTrue(fileNum > store.getStorefiles().size());

  // 3, Split
  requestSplitRegion(regionServer, region, Bytes.toBytes("row3"));
  assertEquals(2, cluster.getRegions(tableName).size());
}
 
Example 19
Source Project: hbase   File: RegexStringComparator.java    License: Apache License 2.0 4 votes vote down vote up
public JoniRegexEngine(String regex, int flags) {
  this.regex = regex;
  byte[] b = Bytes.toBytes(regex);
  this.pattern = new Regex(b, 0, b.length, patternToJoniFlags(flags), encoding, Syntax.Java);
}
 
Example 20
Source Project: metron   File: SaltyRowKeyBuilder.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Builds the 'time' portion of the row key
 * @param period the period
 */
private static byte[] timeKey(long period) {
  return Bytes.toBytes(period);
}