Java Code Examples for org.apache.hadoop.hbase.client.Connection

The following are top voted examples for showing how to use org.apache.hadoop.hbase.client.Connection. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: Oozie_MajorCompaction_Example   File: MajorCompaction.java   Source Code and License 8 votes vote down vote up
public static void main(String[] argc) throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));

  if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
    conf.set("mapreduce.job.credentials.binary",
             System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
  }

  Connection connection = ConnectionFactory.createConnection(conf);
  Admin admin = connection.getAdmin();

  System.out.println("Compacting table " + argc[0]);
  TableName tableName = TableName.valueOf(argc[0]);
  admin.majorCompact(tableName);
  while (admin.getCompactionState(tableName).toString() == "MAJOR") {
    TimeUnit.SECONDS.sleep(10);
    System.out.println("Compacting table " + argc[0]);
  }
  System.out.println("Done compacting table " + argc[0]);
}
 
Example 2
Project: ditb   File: TestTableInputFormat.java   Source Code and License 7 votes vote down vote up
@Override
public void configure(JobConf job) {
  try {
    Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(job));
    TableName tableName = TableName.valueOf("exampleJobConfigurableTable");
    // mandatory
    initializeTable(connection, tableName);
    byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
      Bytes.toBytes("columnB") };
    //optional
    Scan scan = new Scan();
    for (byte[] family : inputColumns) {
      scan.addFamily(family);
    }
    Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
    scan.setFilter(exampleFilter);
    setScan(scan);
  } catch (IOException exception) {
    throw new RuntimeException("Failed to initialize.", exception);
  }
}
 
Example 3
Project: ditb   File: TestRpcClientLeaks.java   Source Code and License 7 votes vote down vote up
@Test(expected=RetriesExhaustedException.class)
public void testSocketClosed() throws IOException, InterruptedException {
  String tableName = "testSocketClosed";
  TableName name = TableName.valueOf(tableName);
  UTIL.createTable(name, fam1).close();

  Configuration conf = new Configuration(UTIL.getConfiguration());
  conf.set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY,
    MyRpcClientImpl.class.getName());
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
  Connection connection = ConnectionFactory.createConnection(conf);
  Table table = connection.getTable(TableName.valueOf(tableName));
  table.get(new Get("asd".getBytes()));
  connection.close();
  for (Socket socket : MyRpcClientImpl.savedSockets) {
    assertTrue("Socket + " +  socket + " is not closed", socket.isClosed());
  }
}
 
Example 4
Project: ditb   File: TestAccessController.java   Source Code and License 7 votes vote down vote up
private void bulkLoadHFile(
    TableName tableName,
    byte[] family,
    byte[] qualifier,
    byte[][][] hfileRanges,
    int numRowsPerRange) throws Exception {

  Path familyDir = new Path(loadPath, Bytes.toString(family));
  fs.mkdirs(familyDir);
  int hfileIdx = 0;
  for (byte[][] range : hfileRanges) {
    byte[] from = range[0];
    byte[] to = range[1];
    createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)),
        family, qualifier, from, to, numRowsPerRange);
  }
  //set global read so RegionServer can move it
  setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));

  try (Connection conn = ConnectionFactory.createConnection(conf);
       HTable table = (HTable)conn.getTable(tableName)) {
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
    loader.doBulkLoad(loadPath, table);
  }
}
 
Example 5
Project: ditb   File: TestCellACLWithMultipleVersions.java   Source Code and License 7 votes vote down vote up
private void verifyUserAllowedforCheckAndDelete(final User user, final byte[] row,
    final byte[] q1, final byte[] value) throws IOException, InterruptedException {
  user.runAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(conf)) {
        try (Table t = connection.getTable(TEST_TABLE.getTableName())) {
          Delete d = new Delete(row);
          d.addColumn(TEST_FAMILY1, q1, 120);
          t.checkAndDelete(row, TEST_FAMILY1, q1, value, d);
        }
      }
      return null;
    }
  });
}
 
Example 6
Project: ditb   File: TestCellACLWithMultipleVersions.java   Source Code and License 7 votes vote down vote up
private void verifyUserDeniedForDeleteExactVersion(final User user, final byte[] row,
    final byte[] q1, final byte[] q2) throws IOException, InterruptedException {
  user.runAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(conf)) {
        try (Table t = connection.getTable(TEST_TABLE.getTableName())) {
          Delete d = new Delete(row, 127);
          d.addColumns(TEST_FAMILY1, q1);
          d.addColumns(TEST_FAMILY1, q2);
          d.addFamily(TEST_FAMILY2, 129);
          t.delete(d);
          fail(user.getShortName() + " can not do the delete");
        } catch (Exception e) {

        }
      }
      return null;
    }
  });
}
 
Example 7
Project: ditb   File: PerformanceEvaluation.java   Source Code and License 6 votes vote down vote up
/**
 * Note that all subclasses of this class must provide a public constructor
 * that has the exact same list of arguments.
 */
Test(final Connection con, final TestOptions options, final Status status) {
  this.connection = con;
  this.conf = con == null ? HBaseConfiguration.create() : this.connection.getConfiguration();
  this.opts = options;
  this.status = status;
  this.testName = this.getClass().getSimpleName();
  receiverHost = SpanReceiverHost.getInstance(conf);
  if (options.traceRate >= 1.0) {
    this.traceSampler = Sampler.ALWAYS;
  } else if (options.traceRate > 0.0) {
    conf.setDouble("hbase.sampler.fraction", options.traceRate);
    this.traceSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(conf));
  } else {
    this.traceSampler = Sampler.NEVER;
  }
  everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
  if (options.isValueZipf()) {
    this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.1);
  }
  LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
}
 
Example 8
Project: ditb   File: TestMasterOperationsForRegionReplicas.java   Source Code and License 6 votes vote down vote up
private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
    int numRegions, int numReplica, Connection connection) throws IOException {
  SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(
    connection);
  snapshot.initialize();
  Map<HRegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
  assert(regionToServerMap.size() == numRegions * numReplica + 1); //'1' for the namespace
  Map<ServerName, List<HRegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
  for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverToRegionMap.entrySet()) {
    if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
      continue;
    }
    List<HRegionInfo> regions = entry.getValue();
    Set<byte[]> setOfStartKeys = new HashSet<byte[]>();
    for (HRegionInfo region : regions) {
      byte[] startKey = region.getStartKey();
      if (region.getTable().equals(table)) {
        setOfStartKeys.add(startKey); //ignore other tables
        LOG.info("--STARTKEY " + new String(startKey)+"--");
      }
    }
    // the number of startkeys will be equal to the number of regions hosted in each server
    // (each server will be hosting one replica of a region)
    assertEquals(numRegions, setOfStartKeys.size());
  }
}
 
Example 9
Project: uavstack   File: DoTestLogData.java   Source Code and License 6 votes vote down vote up
/**
 * [ { "time": 1456293824385, "host": "09-201509070105", "ip": "127.0.0.1", "svrid":
 * "D:/UAV/apache-tomcat-6.0.41::D:/eclipseProject/.metadata/.plugins/org.eclipse.wst.server.core/tmp0", "tag": "L",
 * "frames": { "WebTest": [ { "content": "[CE] aaaaa" } ] } } ]
 */
@SuppressWarnings("unchecked")
public static void testInsertHBase() {

    // MongoDBHandler
    DataStoreMsg msg = new DataStoreMsg();
    String rawData = DataStoreUnitTest.getData(insertJson);
    msg.put(MonitorDataFrame.MessageType.Log.toString(), rawData);
    msg.put(DataStoreProtocol.HBASE_TABLE_NAME, HealthManagerConstants.HBASE_TABLE_LOGDATA);
    List<String> servers = DataConvertHelper.toList(zklist, ",");
    DataStoreConnection obj = new DataStoreConnection(null, null, null, servers, DataStoreType.HBASE);
    obj.putContext(DataStoreProtocol.HBASE_ZK_QUORUM, zklist);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_CACHING, caching);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_MAXRESULTSIZE, maxResultSize);

    obj.putContext(DataStoreProtocol.HBASE_QUERY_REVERSE, true);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_PAGESIZE, 3000);

    AbstractDataStore<Connection> store = DataStoreFactory.getInstance().build(HealthManagerConstants.DataStore_Log,
            obj, new LogDataAdapter(), "");

    store.start();
    boolean rst = store.doInsert(msg);
    store.stop();
    DataStoreUnitTest.printTestResult("testInsertHBase", rst);
}
 
Example 10
Project: uavstack   File: DoTestLogData.java   Source Code and License 6 votes vote down vote up
/**
 * { "starttime": 145629382438, "endtime": 145629382438, //optional "ip": "127.0.0.1", "svrid":
 * "D:/UAV/apache-tomcat-6.0.41::D:/eclipseProject/.metadata/.plugins/org.eclipse.wst.server.core/tmp0", "appid":
 * "sms" }
 */
@SuppressWarnings("unchecked")
public static void testQueryHBase() {

    DataStoreMsg msg = new DataStoreMsg();
    msg.put(DataStoreProtocol.HBASE_QUERY_JSON_KEY, queryJson);
    List<String> servers = DataConvertHelper.toList(zklist, ",");
    DataStoreConnection obj = new DataStoreConnection(null, null, null, servers, DataStoreType.HBASE);
    obj.putContext(DataStoreProtocol.HBASE_ZK_QUORUM, zklist);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_CACHING, caching);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_MAXRESULTSIZE, maxResultSize);
    AbstractDataStore<Connection> store = DataStoreFactory.getInstance().build(HealthManagerConstants.DataStore_Log,
            obj, new LogDataAdapter(), "");
    store.start();
    List<String> rst = store.doQuery(msg);
    store.stop();
    DataStoreUnitTest.printTestResult("testqueryHBase", rst, queryJson);
}
 
Example 11
Project: worm   File: HbaseConnectionWarehouse.java   Source Code and License 6 votes vote down vote up
public Connection getConnection() throws IOException {
	Connection resultConn;
	if (usableSize != 0) {
		resultConn = conns.remove(0);
		usableSize --;
	} else if (currentSize < maxSize) {
		resultConn = ConnectionFactory.createConnection(conf);
		currentSize ++;
	} else {
		try {
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			logger.error("获取HBASE连接出错了!\n"+e.getMessage());
		}
		return this.getConnection();
	}
	return resultConn;
}
 
Example 12
Project: ditb   File: TestAccessController.java   Source Code and License 6 votes vote down vote up
@Test (timeout=180000)
public void testAppend() throws Exception {

  AccessTestAction appendAction = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      byte[] row = TEST_ROW;
      byte[] qualifier = TEST_QUALIFIER;
      Put put = new Put(row);
      put.add(TEST_FAMILY, qualifier, Bytes.toBytes(1));
      Append append = new Append(row);
      append.add(TEST_FAMILY, qualifier, Bytes.toBytes(2));
      try(Connection conn = ConnectionFactory.createConnection(conf);
          Table t = conn.getTable(TEST_TABLE)) {
        t.put(put);
        t.append(append);
      }
      return null;
    }
  };

  verifyAllowed(appendAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE, USER_RW,
    USER_GROUP_WRITE);
  verifyDenied(appendAction, USER_RO, USER_NONE, USER_GROUP_CREATE, USER_GROUP_READ,
    USER_GROUP_ADMIN);
}
 
Example 13
Project: ignite-hbase   File: HBaseCacheStoreTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testManualHBaseInsertion() throws ServiceException, IOException {
  IgniteConfiguration cfg = prepareConfig(false);
  IgniteConfiguration cfg2 = new IgniteConfiguration(cfg);
  cfg.setGridName("first");
  cfg2.setGridName("second");
  String cacheName = "myCache";
  try (Ignite ignite = Ignition.getOrStart(cfg); Ignite ignite2 = Ignition.getOrStart(cfg2)) {
    IgniteCache<String, String> cache = ignite.getOrCreateCache(cacheName);
    cache.remove("Hello");
    assertNull(cache.get("Hello"));
    try (Connection conn = getHBaseConnection()) {
      TableName tableName = TableName.valueOf(TABLE_NAME);
      Table table = conn.getTable(tableName);
      Serializer<Object> serializer = ObjectSerializer.INSTANCE;
      Put put = new Put(serializer.serialize("Hello"));
      put.addColumn(cacheName.getBytes(), QUALIFIER, serializer.serialize("World"));
      table.put(put);
    }
    assertEquals("World", cache.get("Hello"));
  }
}
 
Example 14
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
public static void checkTablePerms(HBaseTestingUtility testUtil, TableName table,
    Permission... perms) throws IOException {
  CheckPermissionsRequest.Builder request = CheckPermissionsRequest.newBuilder();
  for (Permission p : perms) {
    request.addPermission(ProtobufUtil.toPermission(p));
  }

  try(Connection conn = ConnectionFactory.createConnection(testUtil.getConfiguration());
      Table acl = conn.getTable(table)) {
    AccessControlService.BlockingInterface protocol =
      AccessControlService.newBlockingStub(acl.coprocessorService(new byte[0]));
    try {
      protocol.checkPermissions(null, request.build());
    } catch (ServiceException se) {
      ProtobufUtil.toIOException(se);
    }
  }
}
 
Example 15
Project: ditb   File: MetaTableAccessor.java   Source Code and License 6 votes vote down vote up
/**
 * Put the passed <code>ps</code> to the <code>hbase:meta</code> table.
 * @param connection connection we're using
 * @param ps Put to add to hbase:meta
 * @throws IOException
 */
public static void putsToMetaTable(final Connection connection, final List<Put> ps)
  throws IOException {
  Table t = getMetaHTable(connection);
  try {
    t.put(ps);
  } finally {
    t.close();
  }
}
 
Example 16
Project: ditb   File: TestCellACLWithMultipleVersions.java   Source Code and License 6 votes vote down vote up
private void verifyUserDeniedForPutMultipleVersions(final User user, final byte[] row,
    final byte[] q1, final byte[] q2, final byte[] value) throws IOException,
    InterruptedException {
  user.runAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(conf)) {
        try (Table t = connection.getTable(TEST_TABLE.getTableName())) {
          Put p = new Put(row);
          // column Q1 covers version at 123 fr which user2 do not have permission
          p.addColumn(TEST_FAMILY1, q1, 124, value);
          p.addColumn(TEST_FAMILY1, q2, value);
          t.put(p);
          fail(user.getShortName() + " cannot do the put.");
        } catch (Exception e) {

        }
      }
      return null;
    }
  });
}
 
Example 17
Project: ditb   File: IntegrationTestBigLinkedList.java   Source Code and License 6 votes vote down vote up
@Override public int run(String[] args) throws Exception {
  if (args.length < 1) {
    System.err.println("Usage: Clean <output dir>");
    return -1;
  }

  Path p = new Path(args[0]);
  Configuration conf = getConf();
  TableName tableName = getTableName(conf);
  try (FileSystem fs = HFileSystem.get(conf);
      Connection conn = ConnectionFactory.createConnection(conf);
      Admin admin = conn.getAdmin()) {
    if (admin.tableExists(tableName)) {
      admin.disableTable(tableName);
      admin.deleteTable(tableName);
    }

    if (fs.exists(p)) {
      fs.delete(p, true);
    }
  }

  return 0;
}
 
Example 18
Project: ditb   File: TestLoadIncrementalHFiles.java   Source Code and License 6 votes vote down vote up
private void runTest(String testName, HTableDescriptor htd, BloomType bloomType,
    boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception {

  for (boolean managed : new boolean[] { true, false }) {
    Path dir = util.getDataTestDirOnTestFS(testName);
    FileSystem fs = util.getTestFileSystem();
    dir = dir.makeQualified(fs);
    Path familyDir = new Path(dir, Bytes.toString(FAMILY));

    int hfileIdx = 0;
    for (byte[][] range : hfileRanges) {
      byte[] from = range[0];
      byte[] to = range[1];
      HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
          + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
    }
    int expectedRows = hfileIdx * 1000;

    if (preCreateTable) {
      util.getHBaseAdmin().createTable(htd, tableSplitKeys);
    }

    final TableName tableName = htd.getTableName();
    if (!util.getHBaseAdmin().tableExists(tableName)) {
      util.getHBaseAdmin().createTable(htd);
    }
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());

    if (managed) {
      try (HTable table = new HTable(util.getConfiguration(), tableName)) {
        loader.doBulkLoad(dir, table);
        assertEquals(expectedRows, util.countRows(table));
      }
    } else {
      try (Connection conn = ConnectionFactory.createConnection(util.getConfiguration());
          HTable table = (HTable) conn.getTable(tableName)) {
        loader.doBulkLoad(dir, table);
      }
    }

    // verify staging folder has been cleaned up
    Path stagingBasePath = SecureBulkLoadUtil.getBaseStagingDir(util.getConfiguration());
    if (fs.exists(stagingBasePath)) {
      FileStatus[] files = fs.listStatus(stagingBasePath);
      for (FileStatus file : files) {
        assertTrue("Folder=" + file.getPath() + " is not cleaned up.",
            file.getPath().getName() != "DONOTERASE");
      }
    }

    util.deleteTable(tableName);
  }
}
 
Example 19
Project: ditb   File: TestLoadIncrementalHFilesSplitRecovery.java   Source Code and License 6 votes vote down vote up
/**
 * Creates a table with given table name and specified number of column
 * families if the table does not already exist.
 */
private void setupTable(final Connection connection, TableName table, int cfs)
throws IOException {
  try {
    LOG.info("Creating table " + table);
    HTableDescriptor htd = new HTableDescriptor(table);
    for (int i = 0; i < cfs; i++) {
      htd.addFamily(new HColumnDescriptor(family(i)));
    }
    try (Admin admin = connection.getAdmin()) {
      admin.createTable(htd);
    }
  } catch (TableExistsException tee) {
    LOG.info("Table " + table + " already exists");
  }
}
 
Example 20
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Revoke permissions globally from the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeGlobal(final HBaseTestingUtility util, final String user,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.revoke(null, protocol, user, actions);
        }
      }
      return null;
    }
  });
}
 
Example 21
Project: ditb   File: TestTokenAuthentication.java   Source Code and License 6 votes vote down vote up
@Test
public void testUseExistingToken() throws Exception {
  User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "testuser2",
      new String[]{"testgroup"});
  Token<AuthenticationTokenIdentifier> token =
      secretManager.generateToken(user.getName());
  assertNotNull(token);
  user.addToken(token);

  // make sure we got a token
  Token<AuthenticationTokenIdentifier> firstToken =
      new AuthenticationTokenSelector().selectToken(token.getService(), user.getTokens());
  assertNotNull(firstToken);
  assertEquals(token, firstToken);

  Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
  try {
    assertFalse(TokenUtil.addTokenIfMissing(conn, user));
    // make sure we still have the same token
    Token<AuthenticationTokenIdentifier> secondToken =
        new AuthenticationTokenSelector().selectToken(token.getService(), user.getTokens());
    assertEquals(firstToken, secondToken);
  } finally {
    conn.close();
  }
}
 
Example 22
Project: ditb   File: HBaseFsckRepair.java   Source Code and License 6 votes vote down vote up
/**
 * Puts the specified HRegionInfo into META with replica related columns
 */
public static void fixMetaHoleOnlineAndAddReplicas(Configuration conf,
    HRegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException {
  Connection conn = ConnectionFactory.createConnection(conf);
  Table meta = conn.getTable(TableName.META_TABLE_NAME);
  Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
  if (numReplicas > 1) {
    Random r = new Random();
    ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
    for (int i = 1; i < numReplicas; i++) {
      ServerName sn = serversArr[r.nextInt(serversArr.length)];
      // the column added here is just to make sure the master is able to
      // see the additional replicas when it is asked to assign. The
      // final value of these columns will be different and will be updated
      // by the actual regionservers that start hosting the respective replicas
      MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, i);
    }
  }
  meta.put(put);
  meta.close();
  conn.close();
}
 
Example 23
Project: ditb   File: TestVisibilityLabelsWithACL.java   Source Code and License 6 votes vote down vote up
@Test
public void testGetForSuperUserWithFewerLabelAuths() throws Throwable {
  String[] auths = { SECRET };
  String user = "admin";
  VisibilityClient.setAuths(TEST_UTIL.getConnection(), auths, user);
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  final Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
      + "&!" + PRIVATE, SECRET + "&!" + PRIVATE);
  PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
    public Void run() throws Exception {
      Get g = new Get(row1);
      g.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
      try (Connection connection = ConnectionFactory.createConnection(conf);
           Table t = connection.getTable(table.getName())) {
        Result result = t.get(g);
        assertTrue(!result.isEmpty());
      }
      return null;
    }
  };
  SUPERUSER.runAs(scanAction);
}
 
Example 24
Project: ditb   File: TestLoadIncrementalHFilesSplitRecovery.java   Source Code and License 6 votes vote down vote up
/**
 * Checks that all columns have the expected value and that there is the
 * expected number of rows.
 * @throws IOException
 */
void assertExpectedTable(final Connection connection, TableName table, int count, int value)
throws IOException {
  HTableDescriptor [] htds = util.getHBaseAdmin().listTables(table.getNameAsString());
  assertEquals(htds.length, 1);
  Table t = null;
  try {
    t = connection.getTable(table);
    Scan s = new Scan();
    ResultScanner sr = t.getScanner(s);
    int i = 0;
    for (Result r : sr) {
      i++;
      for (NavigableMap<byte[], byte[]> nm : r.getNoVersionMap().values()) {
        for (byte[] val : nm.values()) {
          assertTrue(Bytes.equals(val, value(value)));
        }
      }
    }
    assertEquals(count, i);
  } catch (IOException e) {
    fail("Failed due to exception");
  } finally {
    if (t != null) t.close();
  }
}
 
Example 25
Project: ditb   File: TokenUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Obtain an authentication token for the given user and add it to the
 * user's credentials.
 * @param conn The HBase cluster connection
 * @param user The user for whom to obtain the token
 * @throws IOException If making a remote call to the authentication service fails
 * @throws InterruptedException If executing as the given user is interrupted
 */
public static void obtainAndCacheToken(final Connection conn,
    User user)
    throws IOException, InterruptedException {
  try {
    Token<AuthenticationTokenIdentifier> token = obtainToken(conn, user);

    if (token == null) {
      throw new IOException("No token returned for user " + user.getName());
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("Obtained token " + token.getKind().toString() + " for user " +
          user.getName());
    }
    user.addToken(token);
  } catch (IOException ioe) {
    throw ioe;
  } catch (InterruptedException ie) {
    throw ie;
  } catch (RuntimeException re) {
    throw re;
  } catch (Exception e) {
    throw new UndeclaredThrowableException(e,
        "Unexpected exception obtaining token for user " + user.getName());
  }
}
 
Example 26
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Grant permissions globally to the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void grantGlobal(final HBaseTestingUtility util, final String user,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.grant(null, protocol, user, actions);
        }
      }
      return null;
    }
  });
}
 
Example 27
Project: ditb   File: TestCellACLWithMultipleVersions.java   Source Code and License 6 votes vote down vote up
private void verifyUserDeniedForIncrementMultipleVersions(final User user, final byte[] row,
    final byte[] q1) throws IOException, InterruptedException {
  user.runAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(conf)) {
        try (Table t = connection.getTable(TEST_TABLE.getTableName())) {
          Increment inc = new Increment(row);
          inc.setTimeRange(0, 127);
          inc.addColumn(TEST_FAMILY1, q1, 2L);
          t.increment(inc);
          fail(user.getShortName() + " cannot do the increment.");
        } catch (Exception e) {

        }
      }
      return null;
    }
  });
}
 
Example 28
Project: ditb   File: TableMapReduceUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Obtain an authentication token, for the specified cluster, on behalf of the current user
 * and add it to the credentials for the given map reduce job.
 *
 * @param job The job that requires the permission.
 * @param conf The configuration to use in connecting to the peer cluster
 * @throws IOException When the authentication token cannot be obtained.
 */
public static void initCredentialsForCluster(Job job, Configuration conf)
    throws IOException {
  UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
  if (userProvider.isHBaseSecurityEnabled()) {
    try {
      Connection peerConn = ConnectionFactory.createConnection(conf);
      try {
        TokenUtil.addTokenForJob(peerConn, userProvider.getCurrent(), job);
      } finally {
        peerConn.close();
      }
    } catch (InterruptedException e) {
      LOG.info("Interrupted obtaining user authentication token");
      Thread.interrupted();
    }
  }
}
 
Example 29
Project: ditb   File: TestDefaultScanLabelGeneratorStack.java   Source Code and License 5 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
  // setup configuration
  conf = TEST_UTIL.getConfiguration();
  VisibilityTestUtil.enableVisiblityLabels(conf);
  // Not setting any SLG class. This means to use the default behavior.
  conf.set("hbase.superuser", "admin");
  TEST_UTIL.startMiniCluster(1);
  SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
  TESTUSER = User.createUserForTesting(conf, "test", new String[] { });

  // Wait for the labels table to become available
  TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);

  // Set up for the test
  SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
    public Void run() throws Exception {
      try (Connection conn = ConnectionFactory.createConnection(conf)) {
        VisibilityClient.addLabels(conn, new String[] { SECRET, CONFIDENTIAL });
        VisibilityClient.setAuths(conn, new String[] { CONFIDENTIAL }, TESTUSER.getShortName());
      } catch (Throwable t) {
        throw new IOException(t);
      }
      return null;
    }
  });
}
 
Example 30
Project: ditb   File: RegionSplitter.java   Source Code and License 5 votes vote down vote up
static void createPresplitTable(TableName tableName, SplitAlgorithm splitAlgo,
        String[] columnFamilies, Configuration conf)
throws IOException, InterruptedException {
  final int splitCount = conf.getInt("split.count", 0);
  Preconditions.checkArgument(splitCount > 1, "Split count must be > 1");

  Preconditions.checkArgument(columnFamilies.length > 0,
      "Must specify at least one column family. ");
  LOG.debug("Creating table " + tableName + " with " + columnFamilies.length
      + " column families.  Presplitting to " + splitCount + " regions");

  HTableDescriptor desc = new HTableDescriptor(tableName);
  for (String cf : columnFamilies) {
    desc.addFamily(new HColumnDescriptor(Bytes.toBytes(cf)));
  }
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    Admin admin = connection.getAdmin();
    try {
      Preconditions.checkArgument(!admin.tableExists(tableName),
        "Table already exists: " + tableName);
      admin.createTable(desc, splitAlgo.split(splitCount));
    } finally {
      admin.close();
    }
    LOG.debug("Table created!  Waiting for regions to show online in META...");
    if (!conf.getBoolean("split.verify", true)) {
      // NOTE: createTable is synchronous on the table, but not on the regions
      int onlineRegions = 0;
      while (onlineRegions < splitCount) {
        onlineRegions = MetaTableAccessor.getRegionCount(connection, tableName);
        LOG.debug(onlineRegions + " of " + splitCount + " regions online...");
        if (onlineRegions < splitCount) {
          Thread.sleep(10 * 1000); // sleep
        }
      }
    }
    LOG.debug("Finished creating table with " + splitCount + " regions");
  }
}
 
Example 31
Project: uavstack   File: HBaseDataSource.java   Source Code and License 5 votes vote down vote up
@Override
protected Connection initSourceConnect() throws IOException, ServiceException {

    // 目前只有zklist转成serverlist和dbname
    Configuration config = HBaseConfiguration.create();
    String address = connection.toString(",");
    config.set(DataStoreProtocol.HBASE_ZK_QUORUM, address);
    config.set("hbase.client.scanner.caching",
            (String) connection.getContext(DataStoreProtocol.HBASE_QUERY_CACHING));
    config.set("hbase.client.scanner.max.result.size",
            (String) connection.getContext(DataStoreProtocol.HBASE_QUERY_MAXRESULTSIZE));
    config.set("zookeeper.recovery.retry", String.valueOf(connection.getRetryTimes()));

    // Failed to replace a bad datanode exception protection configuration
    config.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
    config.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");

    HBaseAdmin.checkHBaseAvailable(config);
    conn = ConnectionFactory.createConnection(config);
    // hbase.client.retries.number = 1 and zookeeper.recovery.retry = 1.
    return conn;
}
 
Example 32
Project: ditb   File: TestRegionReplicaFailover.java   Source Code and License 5 votes vote down vote up
/**
 * Tests the case where killing a primary region with unflushed data recovers
 */
@Test (timeout = 120000)
public void testPrimaryRegionKill() throws Exception {
  try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
      Table table = connection.getTable(htd.getTableName())) {

    HTU.loadNumericRows(table, fam, 0, 1000);

    // wal replication is async, we have to wait until the replication catches up, or we timeout
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);

    // we should not have flushed files now, but data in memstores of primary and secondary
    // kill the primary region replica now, and ensure that when it comes back up, we can still
    // read from it the same data from primary and secondaries
    boolean aborted = false;
    for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
      for (Region r : rs.getRegionServer().getOnlineRegions(htd.getTableName())) {
        if (r.getRegionInfo().getReplicaId() == 0) {
          LOG.info("Aborting region server hosting primary region replica");
          rs.getRegionServer().abort("for test");
          aborted = true;
        }
      }
    }
    assertTrue(aborted);

    // wal replication is async, we have to wait until the replication catches up, or we timeout
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 0, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 1, 30000);
    verifyNumericRowsWithTimeout(table, fam, 0, 1000, 2, 30000);
  }

  // restart the region server
  HTU.getMiniHBaseCluster().startRegionServer();
}
 
Example 33
Project: ditb   File: TestScannersWithLabels.java   Source Code and License 5 votes vote down vote up
private static void setAuths() throws Exception {
  String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
  try (Connection conn = ConnectionFactory.createConnection(conf)) {
    VisibilityClient.setAuths(conn, labels, User.getCurrent().getShortName());
  } catch (Throwable t) {
    throw new IOException(t);
  }
}
 
Example 34
Project: ditb   File: MetaTableAccessor.java   Source Code and License 5 votes vote down vote up
/**
 * Splits the region into two in an atomic operation. Offlines the parent
 * region with the information that it is split into two, and also adds
 * the daughter regions. Does not add the location information to the daughter
 * regions since they are not open yet.
 * @param connection connection we're using
 * @param parent the parent region which is split
 * @param splitA Split daughter region A
 * @param splitB Split daughter region A
 * @param sn the location of the region
 */
public static void splitRegion(final Connection connection,
                               HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
                               ServerName sn, int regionReplication) throws IOException {
  Table meta = getMetaHTable(connection);
  try {
    HRegionInfo copyOfParent = new HRegionInfo(parent);
    copyOfParent.setOffline(true);
    copyOfParent.setSplit(true);

    //Put for parent
    Put putParent = makePutFromRegionInfo(copyOfParent);
    addDaughtersToPut(putParent, splitA, splitB);

    //Puts for daughters
    Put putA = makePutFromRegionInfo(splitA);
    Put putB = makePutFromRegionInfo(splitB);

    addLocation(putA, sn, 1, -1, splitA.getReplicaId()); //new regions, openSeqNum = 1 is fine.
    addLocation(putB, sn, 1, -1, splitB.getReplicaId());

    // Add empty locations for region replicas of daughters so that number of replicas can be
    // cached whenever the primary region is looked up from meta
    for (int i = 1; i < regionReplication; i++) {
      addEmptyLocation(putA, i);
      addEmptyLocation(putB, i);
    }

    byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
    multiMutate(meta, tableRow, putParent, putA, putB);
  } finally {
    meta.close();
  }
}
 
Example 35
Project: ditb   File: TestRegionReplicaReplicationEndpoint.java   Source Code and License 5 votes vote down vote up
@Test (timeout = 240000)
public void testRegionReplicaWithoutMemstoreReplication() throws Exception {
  int regionReplication = 3;
  TableName tableName = TableName.valueOf("testRegionReplicaWithoutMemstoreReplication");
  HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
  htd.setRegionReplication(regionReplication);
  htd.setRegionMemstoreReplication(false);
  HTU.getHBaseAdmin().createTable(htd);

  Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
  Table table = connection.getTable(tableName);
  try {
    // write data to the primary. The replicas should not receive the data
    final int STEP = 100;
    for (int i = 0; i < 3; ++i) {
      final int startRow = i * STEP;
      final int endRow = (i + 1) * STEP;
      LOG.info("Writing data from " + startRow + " to " + endRow);
      HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow);
      verifyReplication(tableName, regionReplication, startRow, endRow, false);

      // Flush the table, now the data should show up in the replicas
      LOG.info("flushing table");
      HTU.flush(tableName);
      verifyReplication(tableName, regionReplication, 0, endRow, true);
    }
  } finally {
    table.close();
    connection.close();
  }
}
 
Example 36
Project: dremio-oss   File: TestTableGenerator.java   Source Code and License 5 votes vote down vote up
public static void generateHBaseDatasetDoubleOB(Connection conn, Admin admin, TableName tableName, int numberRegions) throws Exception {
  if (admin.tableExists(tableName)) {
    admin.disableTable(tableName);
    admin.deleteTable(tableName);
  }

  HTableDescriptor desc = new HTableDescriptor(tableName);
  desc.addFamily(new HColumnDescriptor(FAMILY_F));

  if (numberRegions > 1) {
    admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1));
  } else {
    admin.createTable(desc);
  }

  BufferedMutator table = conn.getBufferedMutator(tableName);

  for (double i = 0.5; i <= 100.00; i += 0.75) {
    byte[] bytes = new byte[9];
    PositionedByteRange br = new SimplePositionedMutableByteRange(bytes, 0, 9);
    OrderedBytes.encodeFloat64(br, i, Order.ASCENDING);
    Put p = new Put(bytes);
    p.addColumn(FAMILY_F, COLUMN_C, String.format("value %03f", i).getBytes());
    table.mutate(p);
  }

  table.close();

  admin.flush(tableName);
}
 
Example 37
Project: ditb   File: MultiThreadedClientExample.java   Source Code and License 5 votes vote down vote up
private void warmUpConnectionCache(Connection connection, TableName tn) throws IOException {
  try (RegionLocator locator = connection.getRegionLocator(tn)) {
    LOG.info(
        "Warmed up region location cache for " + tn
            + " got " + locator.getAllRegionLocations().size());
  }
}
 
Example 38
Project: ditb   File: TestCellACLs.java   Source Code and License 5 votes vote down vote up
private void verfifyUserDeniedForWrite(final User user, final byte[] value) throws Exception {
  verifyDenied(new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(conf);
          Table t = connection.getTable(TEST_TABLE.getTableName())) {
        Put p;
        p = new Put(TEST_ROW).addColumn(TEST_FAMILY, TEST_Q1, value);
        t.put(p);
      }
      return null;
    }
  }, user);
}
 
Example 39
Project: ditb   File: TestVisibilityLabelsWithDefaultVisLabelService.java   Source Code and License 5 votes vote down vote up
@Test
public void testListLabels() throws Throwable {
  PrivilegedExceptionAction<ListLabelsResponse> action =
      new PrivilegedExceptionAction<ListLabelsResponse>() {
    public ListLabelsResponse run() throws Exception {
      ListLabelsResponse response = null;
      try (Connection conn = ConnectionFactory.createConnection(conf)) {
        response = VisibilityClient.listLabels(conn, null);
      } catch (Throwable e) {
        fail("Should not have thrown exception");
      }
      // The addLabels() in setup added:
      // { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
      //  UNICODE_VIS_TAG, UC1, UC2 };
      // The previous tests added 2 more labels: ABC, XYZ
      // The 'system' label is excluded.
      List<ByteString> labels = response.getLabelList();
      assertEquals(12, labels.size());
      assertTrue(labels.contains(ByteString.copyFrom(SECRET.getBytes())));
      assertTrue(labels.contains(ByteString.copyFrom(TOPSECRET.getBytes())));
      assertTrue(labels.contains(ByteString.copyFrom(CONFIDENTIAL.getBytes())));
      assertTrue(labels.contains(ByteString.copyFrom("ABC".getBytes())));
      assertTrue(labels.contains(ByteString.copyFrom("XYZ".getBytes())));
      assertFalse(labels.contains(ByteString.copyFrom(SYSTEM_LABEL.getBytes())));
      return null;
    }
  };
  SUPERUSER.runAs(action);
}
 
Example 40
Project: dremio-oss   File: TestTableGenerator.java   Source Code and License 5 votes vote down vote up
public static void generateHBaseDatasetIntOBDesc(Connection conn, Admin admin, TableName tableName, int numberRegions) throws Exception {
  if (admin.tableExists(tableName)) {
    admin.disableTable(tableName);
    admin.deleteTable(tableName);
  }

  HTableDescriptor desc = new HTableDescriptor(tableName);
  desc.addFamily(new HColumnDescriptor(FAMILY_F));

  if (numberRegions > 1) {
    admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1));
  } else {
    admin.createTable(desc);
  }

  BufferedMutator table = conn.getBufferedMutator(tableName);

  for (int i = -49; i <= 100; i ++) {
    byte[] bytes = new byte[5];
    PositionedByteRange br = new SimplePositionedMutableByteRange(bytes, 0, 5);
    OrderedBytes.encodeInt32(br, i, Order.DESCENDING);
    Put p = new Put(bytes);
    p.addColumn(FAMILY_F, COLUMN_C, String.format("value %d", i).getBytes());
    table.mutate(p);
  }

  table.close();

  admin.flush(tableName);
}
 
Example 41
Project: ditb   File: TestVisibilityLabelsWithSLGStack.java   Source Code and License 5 votes vote down vote up
private static void addLabels() throws Exception {
  PrivilegedExceptionAction<VisibilityLabelsResponse> action = 
      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
    public VisibilityLabelsResponse run() throws Exception {
      String[] labels = { SECRET, CONFIDENTIAL };
      try (Connection conn = ConnectionFactory.createConnection(conf)) {
        VisibilityClient.addLabels(conn, labels);
      } catch (Throwable t) {
        throw new IOException(t);
      }
      return null;
    }
  };
  SUPERUSER.runAs(action);
}
 
Example 42
Project: ditb   File: AccessControlLists.java   Source Code and License 5 votes vote down vote up
/**
 * Stores a new user permission grant in the access control lists table.
 * @param conf the configuration
 * @param userPerm the details of the permission to be granted
 * @throws IOException in the case of an error accessing the metadata table
 */
static void addUserPermission(Configuration conf, UserPermission userPerm)
    throws IOException {
  Permission.Action[] actions = userPerm.getActions();
  byte[] rowKey = userPermissionRowKey(userPerm);
  Put p = new Put(rowKey);
  byte[] key = userPermissionKey(userPerm);

  if ((actions == null) || (actions.length == 0)) {
    String msg = "No actions associated with user '" + Bytes.toString(userPerm.getUser()) + "'";
    LOG.warn(msg);
    throw new IOException(msg);
  }

  byte[] value = new byte[actions.length];
  for (int i = 0; i < actions.length; i++) {
    value[i] = actions[i].code();
  }
  p.addImmutable(ACL_LIST_FAMILY, key, value);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Writing permission with rowKey "+
        Bytes.toString(rowKey)+" "+
        Bytes.toString(key)+": "+Bytes.toStringBinary(value)
    );
  }
  // TODO: Pass in a Connection rather than create one each time.
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    try (Table table = connection.getTable(ACL_TABLE_NAME)) {
      table.put(p);
    }
  }
}
 
Example 43
Project: ditb   File: GSScanner.java   Source Code and License 5 votes vote down vote up
public GSScanner(Connection conn, IndexTableRelation relation, Scan scan) throws IOException {
  super(conn, relation, scan);
  rowkeyQueue =
      createRowkeyQueueBySecondaryIndex(conn, relation, scan.getFamilyMap(), rangeList, scan);
  if (rowkeyQueue != null) {
    rawTable = conn.getTable(relation.getTableName());
  } else {
    rawTable = null;
  }
}
 
Example 44
Project: ditb   File: MetaTableAccessor.java   Source Code and License 5 votes vote down vote up
/**
 * Deletes merge qualifiers for the specified merged region.
 * @param connection connection we're using
 * @param mergedRegion
 * @throws IOException
 */
public static void deleteMergeQualifiers(Connection connection,
                                         final HRegionInfo mergedRegion) throws IOException {
  Delete delete = new Delete(mergedRegion.getRegionName());
  delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
  delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
  deleteFromMetaTable(connection, delete);
  LOG.info("Deleted references in merged region "
    + mergedRegion.getRegionNameAsString() + ", qualifier="
    + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
    + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
}
 
Example 45
Project: ditb   File: TestMasterOperationsForRegionReplicas.java   Source Code and License 5 votes vote down vote up
private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
    Connection connection) throws IOException {
  assert(ADMIN.tableExists(table));
  final AtomicInteger count = new AtomicInteger();
  Visitor visitor = new Visitor() {
    @Override
    public boolean visit(Result r) throws IOException {
      if (HRegionInfo.getHRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
      return true;
    }
  };
  MetaTableAccessor.fullScan(connection, visitor);
  assert(count.get() == numRegions);
}
 
Example 46
Project: ditb   File: TableInputFormat.java   Source Code and License 5 votes vote down vote up
@Override
protected Pair<byte[][], byte[][]> getStartEndKeys() throws IOException {
  if (conf.get(SPLIT_TABLE) != null) {
    TableName splitTableName = TableName.valueOf(conf.get(SPLIT_TABLE));
    try (Connection conn = ConnectionFactory.createConnection(getConf())) {
      try (RegionLocator rl = conn.getRegionLocator(splitTableName)) {
        return rl.getStartEndKeys();
      }
    }
  }

  return super.getStartEndKeys();
}
 
Example 47
Project: ditb   File: TableInputFormatBase.java   Source Code and License 5 votes vote down vote up
/**
 * Allows subclasses to initialize the table information.
 *
 * @param connection  The Connection to the HBase cluster. MUST be unmanaged. We will close.
 * @param tableName  The {@link TableName} of the table to process.
 * @throws IOException
 */
protected void initializeTable(Connection connection, TableName tableName) throws IOException {
  if (this.table != null || this.connection != null) {
    LOG.warn("initializeTable called multiple times. Overwriting connection and table " +
        "reference; TableInputFormatBase will not close these old references when done.");
  }
  this.table = (HTable) connection.getTable(tableName);
  this.connection = connection;
}
 
Example 48
Project: ditb   File: TableOutputFormat.java   Source Code and License 5 votes vote down vote up
@Override
public RecordWriter getRecordWriter(FileSystem ignored, JobConf job, String name,
    Progressable progress)
throws IOException {
  // expecting exactly one path
  TableName tableName = TableName.valueOf(job.get(OUTPUT_TABLE));
  BufferedMutator mutator =  null;
  // Connection is not closed. Dies with JVM.  No possibility for cleanup.
  Connection connection = ConnectionFactory.createConnection(job);
  mutator = connection.getBufferedMutator(tableName);
  // Clear write buffer on fail is true by default so no need to reset it.
  return new TableRecordWriter(mutator);
}
 
Example 49
Project: ditb   File: AccessControlClient.java   Source Code and License 5 votes vote down vote up
/**
 * Revoke global permissions for the specified user.
 * @deprecated Use {@link #revoke(Connection, String, Permission.Action...)} instead.
 */
@Deprecated
public static void revoke(Configuration conf, final String userName,
    final Permission.Action... actions) throws Throwable {
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    revoke(connection, userName, actions);
  }
}
 
Example 50
Project: ditb   File: TestRegionReplicaFailover.java   Source Code and License 5 votes vote down vote up
/**
 * Tests the case where we are creating a table with a lot of regions and replicas. Opening region
 * replicas should not block handlers on RS indefinitely.
 */
@Test (timeout = 120000)
public void testLotsOfRegionReplicas() throws IOException {
  int numRegions = NB_SERVERS * 20;
  int regionReplication = 10;
  String tableName = htd.getTableName().getNameAsString() + "2";
  htd = HTU.createTableDescriptor(tableName);
  htd.setRegionReplication(regionReplication);

  // dont care about splits themselves too much
  byte[] startKey = Bytes.toBytes("aaa");
  byte[] endKey = Bytes.toBytes("zzz");
  byte[][] splits = HTU.getRegionSplitStartKeys(startKey, endKey, numRegions);
  HTU.getHBaseAdmin().createTable(htd, startKey, endKey, numRegions);

  try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
      Table table = connection.getTable(htd.getTableName())) {

    for (int i = 1; i < splits.length; i++) {
      for (int j = 0; j < regionReplication; j++) {
        Get get = new Get(splits[i]);
        get.setConsistency(Consistency.TIMELINE);
        get.setReplicaId(j);
        table.get(get); // this should not block. Regions should be coming online
      }
    }
  }

  HTU.deleteTableIfAny(TableName.valueOf(tableName));
}
 
Example 51
Project: ditb   File: QuotaUtil.java   Source Code and License 5 votes vote down vote up
private static void deleteQuotas(final Connection connection, final byte[] rowKey,
    final byte[] qualifier) throws IOException {
  Delete delete = new Delete(rowKey);
  if (qualifier != null) {
    delete.addColumns(QUOTA_FAMILY_INFO, qualifier);
  }
  doDelete(connection, delete);
}
 
Example 52
Project: ditb   File: AccessControlClient.java   Source Code and License 5 votes vote down vote up
/**
 * Revokes the permission on the table for the specified user.
 * @param connection The Connection instance to use
 * @param namespace
 * @param userName
 * @param actions
 * @throws Throwable
 */
public static void revoke(final Connection connection, final String namespace,
    final String userName, final Permission.Action... actions) throws Throwable {
  PayloadCarryingRpcController controller
    = ((ClusterConnection) connection).getRpcControllerFactory().newController();
  try (Table table = connection.getTable(ACL_TABLE_NAME)) {
    ProtobufUtil.revoke(controller, getAccessControlServiceStub(table), userName, namespace,
      actions);
  }
}
 
Example 53
Project: ditb   File: TestVisibilityLabelsWithDeletes.java   Source Code and License 5 votes vote down vote up
@Test
public void testVisibilityLabelsWithDeleteColumns() throws Throwable {
  setAuths();
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());

  try (Table table = createTableAndWriteDataWithLabels(tableName,
      SECRET + "&" + TOPSECRET, SECRET)) {
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility(TOPSECRET + "&" + SECRET));
          d.addColumns(fam, qual);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getHBaseAdmin().flush(tableName);
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 1);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));

  }
}
 
Example 54
Project: ditb   File: SnapshotOfRegionAssignmentFromMeta.java   Source Code and License 5 votes vote down vote up
public SnapshotOfRegionAssignmentFromMeta(Connection connection, Set<TableName> disabledTables,
    boolean excludeOfflinedSplitParents) {
  this.connection = connection;
  tableToRegionMap = new HashMap<TableName, List<HRegionInfo>>();
  regionToRegionServerMap = new HashMap<HRegionInfo, ServerName>();
  regionServerToRegionMap = new HashMap<ServerName, List<HRegionInfo>>();
  regionNameToRegionInfoMap = new TreeMap<String, HRegionInfo>();
  existingAssignmentPlan = new FavoredNodesPlan();
  this.disabledTables = disabledTables;
  this.excludeOfflinedSplitParents = excludeOfflinedSplitParents;
}
 
Example 55
Project: ditb   File: TestScannerResource.java   Source Code and License 5 votes vote down vote up
static int insertData(Configuration conf, TableName tableName, String column, double prob)
    throws IOException {
  Random rng = new Random();
  byte[] k = new byte[3];
  byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
  List<Put> puts = new ArrayList<>();
  for (byte b1 = 'a'; b1 < 'z'; b1++) {
    for (byte b2 = 'a'; b2 < 'z'; b2++) {
      for (byte b3 = 'a'; b3 < 'z'; b3++) {
        if (rng.nextDouble() < prob) {
          k[0] = b1;
          k[1] = b2;
          k[2] = b3;
          Put put = new Put(k);
          put.setDurability(Durability.SKIP_WAL);
          put.add(famAndQf[0], famAndQf[1], k);
          puts.add(put);
        }
      }
    }
  }
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Table table = conn.getTable(tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 
Example 56
Project: ditb   File: PerformanceEvaluation.java   Source Code and License 5 votes vote down vote up
RandomReadTest(Connection con, TestOptions options, Status status) {
  super(con, options, status);
  consistency = options.replicas == DEFAULT_OPTS.replicas ? null : Consistency.TIMELINE;
  if (opts.multiGet > 0) {
    LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
    this.gets = new ArrayList<Get>(opts.multiGet);
  }
}
 
Example 57
Project: ditb   File: ModifyTableProcedure.java   Source Code and License 5 votes vote down vote up
/**
 * update replica column families if necessary.
 * @param env MasterProcedureEnv
 * @throws IOException
 */
private void updateReplicaColumnsIfNeeded(
  final MasterProcedureEnv env,
  final HTableDescriptor oldHTableDescriptor,
  final HTableDescriptor newHTableDescriptor) throws IOException {
  final int oldReplicaCount = oldHTableDescriptor.getRegionReplication();
  final int newReplicaCount = newHTableDescriptor.getRegionReplication();

  if (newReplicaCount < oldReplicaCount) {
    Set<byte[]> tableRows = new HashSet<byte[]>();
    Connection connection = env.getMasterServices().getConnection();
    Scan scan = MetaTableAccessor.getScanForTableName(getTableName());
    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);

    try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME)) {
      ResultScanner resScanner = metaTable.getScanner(scan);
      for (Result result : resScanner) {
        tableRows.add(result.getRow());
      }
      MetaTableAccessor.removeRegionReplicasFromMeta(
        tableRows,
        newReplicaCount,
        oldReplicaCount - newReplicaCount,
        connection);
    }
  }

  // Setup replication for region replicas if needed
  if (newReplicaCount > 1 && oldReplicaCount <= 1) {
    ServerRegionReplicaUtil.setupRegionReplicaReplication(env.getMasterConfiguration());
  }
}
 
Example 58
Project: ditb   File: TokenUtil.java   Source Code and License 5 votes vote down vote up
/**
 * Obtain and return an authentication token for the current user.
 * @param conf the configuration for connecting to the cluster
 * @return the authentication token instance
 * @deprecated Replaced by {@link #obtainToken(Connection)}
 */
@Deprecated
public static Token<AuthenticationTokenIdentifier> obtainToken(
    Configuration conf) throws IOException {
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    return obtainToken(connection);
  }
}
 
Example 59
Project: ditb   File: TableInputFormatBase.java   Source Code and License 5 votes vote down vote up
/**
 * Allows subclasses to initialize the table information.
 *
 * @param connection  The {@link Connection} to the HBase cluster. MUST be unmanaged. We will close.
 * @param tableName  The {@link TableName} of the table to process. 
 * @throws IOException 
 */
protected void initializeTable(Connection connection, TableName tableName) throws IOException {
  if (this.table != null || this.connection != null) {
    LOG.warn("initializeTable called multiple times. Overwriting connection and table " +
        "reference; TableInputFormatBase will not close these old references when done.");
  }
  this.table = connection.getTable(tableName);
  this.regionLocator = connection.getRegionLocator(tableName);
  this.admin = connection.getAdmin();
  this.connection = connection;
}
 
Example 60
Project: ditb   File: TestAccessController.java   Source Code and License 5 votes vote down vote up
private PrivilegedAction<List<UserPermission>> getPrivilegedAction(final String regex) {
  return new PrivilegedAction<List<UserPermission>>() {
    @Override
    public List<UserPermission> run() {
      try(Connection conn = ConnectionFactory.createConnection(conf);) {
        return AccessControlClient.getUserPermissions(conn, regex);
      } catch (Throwable e) {
        LOG.error("error during call of AccessControlClient.getUserPermissions.", e);
        return null;
      }
    }
  };
}