org.apache.hadoop.hbase.client.HConnectionManager Java Examples

The following examples show how to use org.apache.hadoop.hbase.client.HConnectionManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HBaseAction.java    From DDMQ with Apache License 2.0 6 votes vote down vote up
public HBaseAction(ConsumerGroupConfig consumerGroupConfig) {
    group = consumerGroupConfig.getGroup();

    for (String topic : consumerGroupConfig.getTopicMap().keySet()) {
        UpstreamTopic upstreamTopic = consumerGroupConfig.getTopicMap().get(topic);
        if (upstreamTopic.getHbaseconfiguration() != null) {
            // create hbase connection.
            Configuration configuration = HBaseConfiguration.create();
            configuration.set(HbaseConst.ZK_QUORUM, upstreamTopic.getHbaseconfiguration().getHbaseZK());
            configuration.set(HbaseConst.ZK_CLIENT_PORT, upstreamTopic.getHbaseconfiguration().getHbasePort());
            configuration.set(HbaseConst.CLIENT_BUFFER, upstreamTopic.getHbaseconfiguration().getHbaseBuffer());
            configuration.set(HbaseConst.USER_NAME, upstreamTopic.getHbaseconfiguration().getUser());
            configuration.set(HbaseConst.PASSWORD, upstreamTopic.getHbaseconfiguration().getPassword());

            try {
                HBaseConnection HBaseConnection = new HBaseConnection(HConnectionManager.createConnection(configuration));
                connectionMap.put(topic, HBaseConnection);
            } catch (IOException e) {
                LogUtils.logErrorInfo("HBASE_error", "failed to create hbase connection, ", e);
            }
        }
    }
}
 
Example #2
Source File: ThreatHbaseAdapter.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
@Override
public boolean initializeAdapter() {

	// Initialize HBase Table
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);

	try {
		LOGGER.debug("=======Connecting to HBASE===========");
		LOGGER.debug("=======ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));
		HConnection connection = HConnectionManager.createConnection(conf);
		table = connection.getTable(_tableName);
		return true;
	} catch (IOException e) {
		// TODO Auto-generated catch block
		LOGGER.debug("=======Unable to Connect to HBASE===========");
		e.printStackTrace();
	}

	return false;
}
 
Example #3
Source File: CIFHbaseAdapter.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
@Override
public boolean initializeAdapter() {

	// Initialize HBase Table
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);

	try {
		LOGGER.debug("=======Connecting to HBASE===========");
		LOGGER.debug("=======ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));
		HConnection connection = HConnectionManager.createConnection(conf);
		table = connection.getTable(_tableName);
		return true;
	} catch (IOException e) {
		// TODO Auto-generated catch block
		LOGGER.debug("=======Unable to Connect to HBASE===========");
		e.printStackTrace();
	}

	return false;
}
 
Example #4
Source File: HBaseConfigurationUtil.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the cluster conncetion.
 * 
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
private static void createClusterConncetion() throws IOException {
  try {
    if (connectionAvailable()) {
      return;
    }
    clusterConnection = HConnectionManager.createConnection(read());
    addShutdownHook();
    System.out.println("Created HConnection and added shutDownHook");
  } catch (IOException e) {
    LOGGER
        .error(
            "Exception occurred while creating HConnection using HConnectionManager",
            e);
    throw e;
  }
}
 
Example #5
Source File: HBaseConfigurationUtil.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the cluster conncetion.
 * 
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
private static void createClusterConncetion() throws IOException {
  try {
    if (connectionAvailable()) {
      return;
    }
    clusterConnection = HConnectionManager.createConnection(read());
    addShutdownHook();
    System.out.println("Created HConnection and added shutDownHook");
  } catch (IOException e) {
    LOGGER
        .error(
            "Exception occurred while creating HConnection using HConnectionManager",
            e);
    throw e;
  }
}
 
Example #6
Source File: HBaseConnection.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public static HConnection get(String url) {
    // find configuration
    Configuration conf = ConfigCache.get(url);
    if (conf == null) {
        conf = HadoopUtil.newHBaseConfiguration(url);
        ConfigCache.put(url, conf);
    }

    HConnection connection = ConnPool.get(url);
    try {
        // I don't use DCL since recreate a connection is not a big issue.
        if (connection == null) {
            connection = HConnectionManager.createConnection(conf);
            ConnPool.put(url, connection);
        }
    } catch (Throwable t) {
        throw new StorageException("Error when open connection " + url, t);
    }

    return connection;
}
 
Example #7
Source File: Merge.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public Merge(HBaseAdmin admin, Args args) throws IOException {
    if (args.getOptionSet().nonOptionArguments().size() < 2
            || args.getOptionSet().nonOptionArguments().size() > 3) { // todo refactoring
        throw new RuntimeException(Args.INVALID_ARGUMENTS);
    }

    this.admin = admin;
    this.args = args;
    if (args.has(Args.OPTION_TEST)) this.test = true;
    actionParam = (String) args.getOptionSet().nonOptionArguments().get(2);
    this.connection = HConnectionManager.createConnection(admin.getConfiguration());

    tableNameSet = Util.parseTableSet(admin, args);

    if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true;
}
 
Example #8
Source File: Merge.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public Merge(HBaseAdmin admin, Args args) throws IOException {
    if (args.getOptionSet().nonOptionArguments().size() < 2
            || args.getOptionSet().nonOptionArguments().size() > 3) { // todo refactoring
        throw new RuntimeException(Args.INVALID_ARGUMENTS);
    }

    this.admin = admin;
    this.args = args;
    if (args.has(Args.OPTION_TEST)) this.test = true;
    actionParam = (String) args.getOptionSet().nonOptionArguments().get(2);
    this.connection = HConnectionManager.createConnection(admin.getConfiguration());

    tableNameSet = Util.parseTableSet(admin, args);

    if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true;
}
 
Example #9
Source File: Merge.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public Merge(HBaseAdmin admin, Args args) throws IOException {
    if (args.getOptionSet().nonOptionArguments().size() < 2
            || args.getOptionSet().nonOptionArguments().size() > 3) { // todo refactoring
        throw new RuntimeException(Args.INVALID_ARGUMENTS);
    }

    this.admin = admin;
    this.args = args;
    if (args.has(Args.OPTION_TEST)) this.test = true;
    actionParam = (String) args.getOptionSet().nonOptionArguments().get(2);
    this.connection = HConnectionManager.createConnection(admin.getConfiguration());

    tableNameSet = Util.parseTableSet(admin, args);

    if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true;
}
 
Example #10
Source File: Merge.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public Merge(HBaseAdmin admin, Args args) throws IOException {
    if (args.getOptionSet().nonOptionArguments().size() < 2
            || args.getOptionSet().nonOptionArguments().size() > 3) { // todo refactoring
        throw new RuntimeException(Args.INVALID_ARGUMENTS);
    }

    this.admin = admin;
    this.args = args;
    if (args.has(Args.OPTION_TEST)) this.test = true;
    actionParam = (String) args.getOptionSet().nonOptionArguments().get(2);
    this.connection = HConnectionManager.createConnection(admin.getConfiguration());

    tableNameSet = Util.parseTableSet(admin, args);

    if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true;
}
 
Example #11
Source File: Merge.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public Merge(HBaseAdmin admin, Args args) throws IOException {
    if (args.getOptionSet().nonOptionArguments().size() < 2
            || args.getOptionSet().nonOptionArguments().size() > 3) { // todo refactoring
        throw new RuntimeException(Args.INVALID_ARGUMENTS);
    }

    this.admin = admin;
    this.args = args;
    if (args.has(Args.OPTION_TEST)) this.test = true;
    actionParam = (String) args.getOptionSet().nonOptionArguments().get(2);
    this.connection = HConnectionManager.createConnection(admin.getConfiguration());

    tableNameSet = Util.parseTableSet(admin, args);

    if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true;
}
 
Example #12
Source File: HBaseStorageSetup.java    From titan1withtp3.1 with Apache License 2.0 6 votes vote down vote up
public synchronized static void waitForConnection(long timeout, TimeUnit timeoutUnit) {
    long before = System.currentTimeMillis();
    long after;
    long timeoutMS = TimeUnit.MILLISECONDS.convert(timeout, timeoutUnit);
    do {
        try {
            HConnection hc = HConnectionManager.createConnection(HBaseConfiguration.create());
            hc.close();
            after = System.currentTimeMillis();
            log.info("HBase server to started after about {} ms", after - before);
            return;
        } catch (IOException e) {
            log.info("Exception caught while waiting for the HBase server to start", e);
        }
        after = System.currentTimeMillis();
    } while (timeoutMS > after - before);
    after = System.currentTimeMillis();
    log.warn("HBase server did not start in {} ms", after - before);
}
 
Example #13
Source File: DataJanitorStateTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Before
public void beforeTest() throws Exception {
  pruneStateTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                               TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
  HTable table = createTable(pruneStateTable.getName(), new byte[][]{DataJanitorState.FAMILY}, false,
                             // Prune state table is a non-transactional table, hence no transaction co-processor
                             Collections.<String>emptyList());
  table.close();
  connection = HConnectionManager.createConnection(conf);

  dataJanitorState =
    new DataJanitorState(new DataJanitorState.TableSupplier() {
      @Override
      public HTableInterface get() throws IOException {
        return connection.getTable(pruneStateTable);
      }
    });

}
 
Example #14
Source File: DataJanitorStateTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Before
public void beforeTest() throws Exception {
  pruneStateTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                               TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
  HTable table = createTable(pruneStateTable.getName(), new byte[][]{DataJanitorState.FAMILY}, false,
                             // Prune state table is a non-transactional table, hence no transaction co-processor
                             Collections.<String>emptyList());
  table.close();
  connection = HConnectionManager.createConnection(conf);

  dataJanitorState =
    new DataJanitorState(new DataJanitorState.TableSupplier() {
      @Override
      public HTableInterface get() throws IOException {
        return connection.getTable(pruneStateTable);
      }
    });

}
 
Example #15
Source File: HBaseTransactionPruningPlugin.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(Configuration conf) throws IOException {
  this.conf = conf;
  this.hBaseAdmin = new HBaseAdmin(conf);
  this.connection = HConnectionManager.createConnection(conf);

  final TableName stateTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                                          TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
  LOG.info("Initializing plugin with state table {}:{}", stateTable.getNamespaceAsString(),
           stateTable.getNameAsString());
  createPruneTable(stateTable);
  this.dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
    @Override
    public HTableInterface get() throws IOException {
      return connection.getTable(stateTable);
    }
  });
}
 
Example #16
Source File: HBaseTransactionPruningPlugin.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(Configuration conf) throws IOException {
  this.conf = conf;
  this.hBaseAdmin = new HBaseAdmin(conf);
  this.connection = HConnectionManager.createConnection(conf);

  final TableName stateTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                                          TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
  LOG.info("Initializing plugin with state table {}:{}", stateTable.getNamespaceAsString(),
           stateTable.getNameAsString());
  createPruneTable(stateTable);
  this.dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
    @Override
    public HTableInterface get() throws IOException {
      return connection.getTable(stateTable);
    }
  });
}
 
Example #17
Source File: BalanceBooks.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
/**
 * Sets up common resources required by all clients.
 */
public void init() throws IOException {
  Injector injector = Guice.createInjector(
      new ConfigModule(conf),
      new ZKModule(),
      new DiscoveryModules().getDistributedModules(),
      new TransactionModules().getDistributedModules(),
      new TransactionClientModule()
  );

  zkClient = injector.getInstance(ZKClientService.class);
  zkClient.startAndWait();
  txClient = injector.getInstance(TransactionServiceClient.class);

  createTableIfNotExists(conf, TABLE, new byte[][]{ FAMILY });
  conn = HConnectionManager.createConnection(conf);
}
 
Example #18
Source File: HBaseAction.java    From DDMQ with Apache License 2.0 6 votes vote down vote up
public HBaseAction(ConsumerGroupConfig consumerGroupConfig) {
    group = consumerGroupConfig.getGroup();

    for (String topic : consumerGroupConfig.getTopicMap().keySet()) {
        UpstreamTopic upstreamTopic = consumerGroupConfig.getTopicMap().get(topic);
        if (upstreamTopic.getHbaseconfiguration() != null) {
            // create hbase connection.
            Configuration configuration = HBaseConfiguration.create();
            configuration.set(HbaseConst.ZK_QUORUM, upstreamTopic.getHbaseconfiguration().getHbaseZK());
            configuration.set(HbaseConst.ZK_CLIENT_PORT, upstreamTopic.getHbaseconfiguration().getHbasePort());
            configuration.set(HbaseConst.CLIENT_BUFFER, upstreamTopic.getHbaseconfiguration().getHbaseBuffer());
            configuration.set(HbaseConst.USER_NAME, upstreamTopic.getHbaseconfiguration().getUser());
            configuration.set(HbaseConst.PASSWORD, upstreamTopic.getHbaseconfiguration().getPassword());

            try {
                HBaseConnection HBaseConnection = new HBaseConnection(HConnectionManager.createConnection(configuration));
                connectionMap.put(topic, HBaseConnection);
            } catch (IOException e) {
                LogUtils.logErrorInfo("HBASE_error", "failed to create hbase connection, ", e);
            }
        }
    }
}
 
Example #19
Source File: HBaseSimpleDemo.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public HConnection openConn() {
	if (null == hconn) {
		try {
			this.hconn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}
	return hconn;
}
 
Example #20
Source File: InvalidListPruningDebugTool.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
/**
 * Initialize the Invalid List Debug Tool.
 * @param conf {@link Configuration}
 * @throws IOException when not able to create an HBase connection
 */
@Override
@SuppressWarnings("WeakerAccess")
public void initialize(final Configuration conf) throws IOException {
  LOG.debug("InvalidListPruningDebugMain : initialize method called");
  connection = HConnectionManager.createConnection(conf);
  tableName = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                         TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
  dataJanitorState = new DataJanitorState(new DataJanitorState.TableSupplier() {
    @Override
    public HTableInterface get() throws IOException {
      return connection.getTable(tableName);
    }
  });
}
 
Example #21
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void startMiniCluster() throws Exception {
  // Setup the configuration to start HBase cluster with the invalid list pruning enabled
  conf = HBaseConfiguration.create();
  conf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, true);
  // Flush prune data to table quickly, so that tests don't need have to wait long to see updates
  conf.setLong(TxConstants.TransactionPruning.PRUNE_FLUSH_INTERVAL, 0L);
  AbstractHBaseTableTest.startMiniCluster();

  TransactionStateStorage txStateStorage = new InMemoryTransactionStateStorage();
  TransactionManager txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
  txManager.startAndWait();

  // Do some transactional data operations
  txDataTable1 = TableName.valueOf("invalidListPruneTestTable1");
  HTable hTable = createTable(txDataTable1.getName(), new byte[][]{family}, false,
                              Collections.singletonList(TestTransactionProcessor.class.getName()));
  try (TransactionAwareHTable txTable = new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
    TransactionContext txContext = new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
    txContext.start();
    for (int i = 0; i < MAX_ROWS; ++i) {
      txTable.put(new Put(Bytes.toBytes(i)).add(family, qualifier, Bytes.toBytes(i)));
    }
    txContext.finish();
  }

  testUtil.flush(txDataTable1);
  txManager.stopAndWait();

  pruneStateTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                                               TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
  connection = HConnectionManager.createConnection(conf);
  dataJanitorState =
    new DataJanitorState(new DataJanitorState.TableSupplier() {
      @Override
      public HTableInterface get() throws IOException {
        return connection.getTable(pruneStateTable);
      }
    });
}
 
Example #22
Source File: WhoisHBaseAdapter.java    From opensoc-streaming with Apache License 2.0 5 votes vote down vote up
public boolean initializeAdapter() {
	Configuration conf = null;
	conf = HBaseConfiguration.create();
	conf.set("hbase.zookeeper.quorum", _quorum);
	conf.set("hbase.zookeeper.property.clientPort", _port);
	conf.set("zookeeper.session.timeout", "20");
	conf.set("hbase.rpc.timeout", "20");
	conf.set("zookeeper.recovery.retry", "1");
	conf.set("zookeeper.recovery.retry.intervalmill", "1");

	try {

		LOG.trace("[OpenSOC] Connecting to HBase");
		LOG.trace("[OpenSOC] ZOOKEEPER = "
				+ conf.get("hbase.zookeeper.quorum"));

		LOG.trace("[OpenSOC] CONNECTING TO HBASE WITH: " + conf);

		HConnection connection = HConnectionManager.createConnection(conf);

		LOG.trace("[OpenSOC] CONNECTED TO HBASE");

		table = connection.getTable(_table_name);

		LOG.trace("--------CONNECTED TO TABLE: " + table);

		JSONObject tester = enrich("cisco.com");

		if (tester.keySet().size() == 0)
			throw new IOException(
					"Either HBASE is misconfigured or whois table is missing");

		return true;
	} catch (IOException e) {
		e.printStackTrace();
	}

	return false;

}
 
Example #23
Source File: HBaseTablespace.java    From tajo with Apache License 2.0 5 votes vote down vote up
public HConnection getConnection() throws IOException {
  synchronized(connMap) {
    HConnectionKey key = new HConnectionKey(hbaseConf);
    HConnection conn = connMap.get(key);
    if (conn == null) {
      conn = HConnectionManager.createConnection(hbaseConf);
      connMap.put(key, conn);
    }

    return conn;
  }
}
 
Example #24
Source File: TwoLevelIndexBuilder.java    From yuzhouwan with Apache License 2.0 5 votes vote down vote up
private TwoLevelIndexBuilder(String rootDir, String zkServer, String port) throws IOException {
    conf = HBaseConfiguration.create();
    conf.set("hbase.rootdir", rootDir);
    conf.set("hbase.zookeeper.quorum", zkServer);
    conf.set("hbase.zookeeper.property.clientPort", port);

    HConnectionManager.createConnection(conf);
}
 
Example #25
Source File: HBaseFactoryTest.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public HConnection openConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}
	return conn;
}
 
Example #26
Source File: HBaseConnAbstractPool.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public synchronized HConnection getConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(this.config);
		} catch (Exception ex) {
			LOGGER.error("create conn err:", ex);
		}
	}
	return conn;
}
 
Example #27
Source File: HBaseClientManager.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public synchronized HConnection getConn() {
	if (null == conn) {
		try {
			this.conn = HConnectionManager.createConnection(config);
		} catch (Exception ex) {
			LOGGER.error("create conn err:", ex);
		}
	}
	return conn;
}
 
Example #28
Source File: HConnectionController.java    From recsys-offline with Apache License 2.0 5 votes vote down vote up
public  HConnection getHConnection() {
    HConnection con = null;
    try {
        if (null == con || con.isClosed()) {
            con = HConnectionManager.createConnection(config);
        }
    } catch (IOException e) {
        logger.error("Cannot get HBase connection.", e.getCause());
    }
    return con;
}
 
Example #29
Source File: HBasePool.java    From DistributedCrawler with Apache License 2.0 5 votes vote down vote up
private HBasePool() throws ZooKeeperConnectionException {
	Configuration conf = new Configuration();
	conf.set("hbase.zookeeper.property.clientPort", "2181");
	conf.set("hbase.zookeeper.quorum", "gs-pc");
	conf.set("hbase.master", "gs-pc:60000");
	this.conf = conf;
	connection = HConnectionManager.createConnection(conf);
}
 
Example #30
Source File: TestHBaseDAO.java    From DistributedCrawler with Apache License 2.0 5 votes vote down vote up
@Test
public void testPut() throws IOException {
	Configuration configuration = HBaseConfiguration.create();
	HConnection connection = HConnectionManager.createConnection(configuration);
	 HTableInterface table = connection.getTable("page");
	 // use the table as needed, for a single operation and a single thread
	 Put put = new Put("2".getBytes());
	 put.add("content".getBytes(), null, "我吃包子".getBytes());
	 put.add("title".getBytes(), null, "吃包子".getBytes());
	 put.add("url".getBytes(), null, "http://www.sina.com.cn".getBytes());
	 table.put(put);
	 table.close();
	 connection.close();
}