org.apache.hadoop.hbase.HBaseConfiguration Java Examples
The following examples show how to use
org.apache.hadoop.hbase.HBaseConfiguration.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HfileBulkImporter.java From super-cloudops with Apache License 2.0 | 7 votes |
/** * e.g.</br> * * <pre> * yarn jar super-devops-tool-hbase-migrator-master.jar \ * com.wl4g.devops.tool.hbase.migrator.HfileBulkImporter \ * -z emr-header-1:2181 \ * -t safeclound.tb_elec_power \ * -p /tmp-devops/safeclound.tb_elec_power * </pre> * * @param args * @throws Exception */ public static void main(String[] args) throws Exception { HbaseMigrateUtils.showBanner(); CommandLine line = new Builder().option("z", "zkaddr", null, "Zookeeper address.") .option("t", "tabname", null, "Hbase table name.") .option("p", "path", null, "Data hdfs path to be import. e.g. hdfs://localhost:9000/bak/safeclound.tb_air") .build(args); Configuration cfg = HBaseConfiguration.create(); cfg.set("hbase.zookeeper.quorum", line.getOptionValue("z")); Connection conn = ConnectionFactory.createConnection(cfg); Admin admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf(line.getOptionValue("t"))); LoadIncrementalHFiles load = new LoadIncrementalHFiles(cfg); load.doBulkLoad(new Path(line.getOptionValue("p")), admin, table, conn.getRegionLocator(TableName.valueOf(line.getOptionValue("t")))); }
Example #2
Source File: HBaseIndex.java From hudi with Apache License 2.0 | 6 votes |
private Connection getHBaseConnection() { Configuration hbaseConfig = HBaseConfiguration.create(); String quorum = config.getHbaseZkQuorum(); hbaseConfig.set("hbase.zookeeper.quorum", quorum); String zkZnodeParent = config.getHBaseZkZnodeParent(); if (zkZnodeParent != null) { hbaseConfig.set("zookeeper.znode.parent", zkZnodeParent); } String port = String.valueOf(config.getHbaseZkPort()); hbaseConfig.set("hbase.zookeeper.property.clientPort", port); try { return ConnectionFactory.createConnection(hbaseConfig); } catch (IOException e) { throw new HoodieDependentSystemUnavailableException(HoodieDependentSystemUnavailableException.HBASE, quorum + ":" + port); } }
Example #3
Source File: TestTableInputFormat.java From hbase with Apache License 2.0 | 6 votes |
@Override protected void initialize(JobContext job) throws IOException { Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create( job.getConfiguration())); TableName tableName = TableName.valueOf("exampleTable"); // mandatory initializeTable(connection, tableName); byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"), Bytes.toBytes("columnB") }; //optional Scan scan = new Scan(); for (byte[] family : inputColumns) { scan.addFamily(family); } Filter exampleFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*")); scan.setFilter(exampleFilter); setScan(scan); }
Example #4
Source File: IndexUpgradeTool.java From phoenix with Apache License 2.0 | 6 votes |
@VisibleForTesting public int executeTool() { Configuration conf = HBaseConfiguration.addHbaseResources(getConf()); try (Connection conn = getConnection(conf)) { ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class) .getQueryServices(); boolean status = extractTablesAndIndexes(conn.unwrap(PhoenixConnection.class)); if (status) { return executeTool(conn, queryServices, conf); } } catch (SQLException e) { LOGGER.severe("Something went wrong in executing tool "+ e); } return -1; }
Example #5
Source File: IndexScrutinyTool.java From phoenix with Apache License 2.0 | 6 votes |
private Job configureSubmittableJob(Job job, Path outputPath, Class<IndexScrutinyMapperForTest> mapperClass) throws Exception { Configuration conf = job.getConfiguration(); conf.setBoolean("mapreduce.job.user.classpath.first", true); HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf)); job.setJarByClass(IndexScrutinyTool.class); job.setOutputFormatClass(NullOutputFormat.class); if (outputInvalidRows && OutputFormat.FILE.equals(outputFormat)) { job.setOutputFormatClass(TextOutputFormat.class); FileOutputFormat.setOutputPath(job, outputPath); } job.setMapperClass((mapperClass == null ? IndexScrutinyMapper.class : mapperClass)); job.setNumReduceTasks(0); // Set the Output classes job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); TableMapReduceUtil.addDependencyJars(job); return job; }
Example #6
Source File: HBaseRangerAuthorizationTest.java From ranger with Apache License 2.0 | 6 votes |
@Test public void testReadRowFromColFam2AsGroupIT() throws Exception { final Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", "localhost"); conf.set("hbase.zookeeper.property.clientPort", "" + port); conf.set("zookeeper.znode.parent", "/hbase-unsecure"); String user = "public"; UserGroupInformation ugi = UserGroupInformation.createUserForTesting(user, new String[] {"IT"}); ugi.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { Connection conn = ConnectionFactory.createConnection(conf); Table table = conn.getTable(TableName.valueOf("temp")); // Read a row Get get = new Get(Bytes.toBytes("row1")); Result result = table.get(get); byte[] valResult = result.getValue(Bytes.toBytes("colfam2"), Bytes.toBytes("col1")); Assert.assertNull(valResult); conn.close(); return null; } }); }
Example #7
Source File: TestHbaseClient.java From Kylin with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws IOException { foo(6, 5); foo(5, 2); foo(3, 0); Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", "hbase_host"); conf.set("zookeeper.znode.parent", "/hbase-unsecure"); HTable table = new HTable(conf, "test1"); Put put = new Put(Bytes.toBytes("row1")); put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1")); put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val2")); table.put(put); table.close(); }
Example #8
Source File: ReplicationSink.java From hbase with Apache License 2.0 | 6 votes |
/** * Create a sink for replication * @param conf conf object * @param stopper boolean to tell this thread to stop * @throws IOException thrown when HDFS goes bad or bad file name */ public ReplicationSink(Configuration conf, Stoppable stopper) throws IOException { this.conf = HBaseConfiguration.create(conf); decorateConf(); this.metrics = new MetricsSink(); this.walEntrySinkFilter = setupWALEntrySinkFilter(); String className = conf.get("hbase.replication.source.fs.conf.provider", DefaultSourceFSConfigurationProvider.class.getCanonicalName()); try { Class<? extends SourceFSConfigurationProvider> c = Class.forName(className).asSubclass(SourceFSConfigurationProvider.class); this.provider = c.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new IllegalArgumentException( "Configured source fs configuration provider class " + className + " throws error.", e); } }
Example #9
Source File: TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java From hbase with Apache License 2.0 | 6 votes |
@BeforeClass public static void beforeAllTests() throws Exception { groups = new String[] { RSGroupInfo.DEFAULT_GROUP }; servers = generateServers(3); groupMap = constructGroupInfo(servers, groups); tableDescs = constructTableDesc(false); Configuration conf = HBaseConfiguration.create(); conf.set("hbase.regions.slop", "0"); conf.setFloat("hbase.master.balancer.stochastic.readRequestCost", 10000f); conf.set("hbase.rsgroup.grouploadbalancer.class", StochasticLoadBalancer.class.getCanonicalName()); loadBalancer = new RSGroupBasedLoadBalancer(); loadBalancer.setRsGroupInfoManager(getMockedGroupInfoManager()); loadBalancer.setMasterServices(getMockedMaster()); loadBalancer.setConf(conf); loadBalancer.initialize(); }
Example #10
Source File: TestThriftConnection.java From hbase with Apache License 2.0 | 6 votes |
private static ThriftServer startThriftServer(int port, boolean useHttp) { Configuration thriftServerConf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); thriftServerConf.setInt(Constants.PORT_CONF_KEY, port); if (useHttp) { thriftServerConf.setBoolean(Constants.USE_HTTP_CONF_KEY, true); } ThriftServer server = new ThriftServer(thriftServerConf); Thread thriftServerThread = new Thread(() -> { try{ server.run(); } catch (Exception t) { LOG.error("Thrift Server failed", t); } }); thriftServerThread.setDaemon(true); thriftServerThread.start(); if (useHttp) { TEST_UTIL.waitFor(10000, () -> server.getHttpServer() != null); } else { TEST_UTIL.waitFor(10000, () -> server.getTserver() != null); } return server; }
Example #11
Source File: HBaseDao.java From metron with Apache License 2.0 | 6 votes |
@Override public synchronized void init(AccessConfig config) { if(this.tableInterface == null) { this.config = config; Map<String, Object> globalConfig = config.getGlobalConfigSupplier().get(); if(globalConfig == null) { throw new IllegalStateException("Cannot find the global config."); } String table = (String)globalConfig.get(HBASE_TABLE); String cf = (String) config.getGlobalConfigSupplier().get().get(HBASE_CF); if(table == null || cf == null) { throw new IllegalStateException("You must configure " + HBASE_TABLE + " and " + HBASE_CF + " in the global config."); } try { tableInterface = config.getTableProvider().getTable(HBaseConfiguration.create(), table); this.cf = cf.getBytes(StandardCharsets.UTF_8); } catch (IOException e) { throw new IllegalStateException("Unable to initialize HBaseDao: " + e.getMessage(), e); } } }
Example #12
Source File: TestThriftConnection.java From hbase with Apache License 2.0 | 6 votes |
private static Connection createConnection(int port, boolean useHttp) throws IOException { Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); conf.set(ConnectionUtils.HBASE_CLIENT_CONNECTION_IMPL, ThriftConnection.class.getName()); if (useHttp) { conf.set(Constants.HBASE_THRIFT_CLIENT_BUIDLER_CLASS, ThriftConnection.HTTPThriftClientBuilder.class.getName()); } String host = HConstants.LOCALHOST; if (useHttp) { host = "http://" + host; } conf.set(Constants.HBASE_THRIFT_SERVER_NAME, host); conf.setInt(Constants.HBASE_THRIFT_SERVER_PORT, port); return ConnectionFactory.createConnection(conf); }
Example #13
Source File: Main.java From flink-learning with Apache License 2.0 | 6 votes |
private static void writeEventToHbase(String string, ParameterTool parameterTool) throws IOException { Configuration configuration = HBaseConfiguration.create(); configuration.set(HBASE_ZOOKEEPER_QUORUM, parameterTool.get(HBASE_ZOOKEEPER_QUORUM)); configuration.set(HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT, parameterTool.get(HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT)); configuration.set(HBASE_RPC_TIMEOUT, parameterTool.get(HBASE_RPC_TIMEOUT)); configuration.set(HBASE_CLIENT_OPERATION_TIMEOUT, parameterTool.get(HBASE_CLIENT_OPERATION_TIMEOUT)); configuration.set(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, parameterTool.get(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)); Connection connect = ConnectionFactory.createConnection(configuration); Admin admin = connect.getAdmin(); if (!admin.tableExists(HBASE_TABLE_NAME)) { //检查是否有该表,如果没有,创建 admin.createTable(new HTableDescriptor(HBASE_TABLE_NAME).addFamily(new HColumnDescriptor(INFO_STREAM))); } Table table = connect.getTable(HBASE_TABLE_NAME); TimeStamp ts = new TimeStamp(new Date()); Date date = ts.getDate(); Put put = new Put(Bytes.toBytes(date.getTime())); put.addColumn(Bytes.toBytes(INFO_STREAM), Bytes.toBytes("test"), Bytes.toBytes(string)); table.put(put); table.close(); connect.close(); }
Example #14
Source File: DropIndexDuringUpsertIT.java From phoenix with Apache License 2.0 | 6 votes |
@Before public void doSetup() throws Exception { Configuration conf = HBaseConfiguration.create(); setUpConfigForMiniCluster(conf); conf.setInt("hbase.client.retries.number", 2); conf.setInt("hbase.client.pause", 5000); conf.setInt("hbase.balancer.period", Integer.MAX_VALUE); conf.setLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, 0); util = new HBaseTestingUtility(conf); util.startMiniCluster(NUM_SLAVES); String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB); url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM; Map<String, String> props = Maps.newHashMapWithExpectedSize(1); // Must update config before starting server props.put(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); driver = initAndRegisterTestDriver(url, new ReadOnlyProps(props.entrySet().iterator())); }
Example #15
Source File: HBaseStreamPartitioner.java From opensoc-streaming with Apache License 2.0 | 6 votes |
public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> targetTasks) { System.out.println("preparing HBaseStreamPartitioner for streamId " + stream.get_streamId()); this.targetTasks = targetTasks; this.targetTasksSize = this.targetTasks.size(); Configuration conf = HBaseConfiguration.create(); try { hTable = new HTable(conf, tableName); refreshRegionInfo(tableName); System.out.println("regionStartKeyRegionNameMap: " + regionStartKeyRegionNameMap); } catch (IOException e) { e.printStackTrace(); } }
Example #16
Source File: BackupSystemTable.java From hbase with Apache License 2.0 | 6 votes |
/** * Get backup system table descriptor * @return table's descriptor */ public static TableDescriptor getSystemTableForBulkLoadedDataDescriptor(Configuration conf) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableNameForBulkLoadedData(conf)); ColumnFamilyDescriptorBuilder colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY); colBuilder.setMaxVersions(1); Configuration config = HBaseConfiguration.create(); int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY, BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT); colBuilder.setTimeToLive(ttl); ColumnFamilyDescriptor colSessionsDesc = colBuilder.build(); builder.setColumnFamily(colSessionsDesc); colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY); colBuilder.setTimeToLive(ttl); builder.setColumnFamily(colBuilder.build()); return builder.build(); }
Example #17
Source File: TestHBaseBase.java From eagle with Apache License 2.0 | 6 votes |
@BeforeClass public static void setUpHBase() { Configuration configuration = HBaseConfiguration.create(); configuration.set("zookeeper.znode.parent", getZkZnodeParent()); configuration.setInt("hbase.master.info.port", -1);//avoid port clobbering configuration.setInt("hbase.regionserver.info.port", -1);//avoid port clobbering hbase = new HBaseTestingUtility(configuration); try { hbase.startMiniCluster(); } catch (Exception e) { LOGGER.error("Error to start hbase mini cluster: " + e.getMessage(), e); throw new IllegalStateException(e); } System.setProperty("storage.hbase.autoCreateTable","false"); System.setProperty("storage.hbase.zookeeperZnodeParent", getZkZnodeParent()); System.setProperty("storage.hbase.zookeeperPropertyClientPort", String.valueOf(hbase.getZkCluster().getClientPort())); }
Example #18
Source File: TestExecutorService.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testSnapshotHandlers() throws Exception { final Configuration conf = HBaseConfiguration.create(); final Server server = mock(Server.class); when(server.getConfiguration()).thenReturn(conf); ExecutorService executorService = new ExecutorService("testSnapshotHandlers"); executorService.startExecutorService(ExecutorType.MASTER_SNAPSHOT_OPERATIONS, 1); CountDownLatch latch = new CountDownLatch(1); CountDownLatch waitForEventToStart = new CountDownLatch(1); executorService.submit(new EventHandler(server, EventType.C_M_SNAPSHOT_TABLE) { @Override public void process() throws IOException { waitForEventToStart.countDown(); try { latch.await(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } }); //Wait EventHandler to start waitForEventToStart.await(10, TimeUnit.SECONDS); int activeCount = executorService.getExecutor(ExecutorType.MASTER_SNAPSHOT_OPERATIONS) .getThreadPoolExecutor().getActiveCount(); Assert.assertEquals(1, activeCount); latch.countDown(); Waiter.waitFor(conf, 3000, () -> { int count = executorService.getExecutor(ExecutorType.MASTER_SNAPSHOT_OPERATIONS) .getThreadPoolExecutor().getActiveCount(); return count == 0; }); }
Example #19
Source File: UpdateStatisticsToolTest.java From phoenix with Apache License 2.0 | 5 votes |
@Test public void testRestoreDirFromConfig() { UpdateStatisticsTool tool = new UpdateStatisticsTool(); Configuration configuration = HBaseConfiguration.create(); configuration.set(FS_DEFAULT_NAME_KEY, "hdfs://base-dir"); tool.setConf(configuration); tool.parseArgs(new String[] {"-t", "table1", "-ms", "-runfg"}); assertEquals("hdfs://base-dir/tmp", tool.getRestoreDir().toString()); }
Example #20
Source File: HBaseTestHelper.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
private static Configuration getConfiguration() { Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", "127.0.0.1"); conf.set("hbase.zookeeper.property.clientPort", "2181"); return conf; }
Example #21
Source File: ThriftServer.java From hbase with Apache License 2.0 | 5 votes |
public static void main(String [] args) throws Exception { LOG.info("***** STARTING service '" + ThriftServer.class.getSimpleName() + "' *****"); VersionInfo.logVersion(); final Configuration conf = HBaseConfiguration.create(); // for now, only time we return is on an argument error. final int status = ToolRunner.run(conf, new ThriftServer(conf), args); LOG.info("***** STOPPING service '" + ThriftServer.class.getSimpleName() + "' *****"); System.exit(status); }
Example #22
Source File: TestSimpleRequestController.java From hbase with Apache License 2.0 | 5 votes |
private void testIllegalArgument(String key, long value) { Configuration conf = HBaseConfiguration.create(); conf.setLong(key, value); try { new SimpleRequestController(conf); fail("The " + key + " must be bigger than zero"); } catch (IllegalArgumentException e) { } }
Example #23
Source File: TestSimpleRpcScheduler.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testScanQueueWithZeroScanRatio() throws Exception { Configuration schedConf = HBaseConfiguration.create(); schedConf.setFloat(RpcExecutor.CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 1.0f); schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_READ_SHARE_CONF_KEY, 0.5f); schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_SCAN_SHARE_CONF_KEY, 0f); PriorityFunction priority = mock(PriorityFunction.class); when(priority.getPriority(any(), any(), any())).thenReturn(HConstants.NORMAL_QOS); RpcScheduler scheduler = new SimpleRpcScheduler(schedConf, 2, 1, 1, priority, HConstants.QOS_THRESHOLD); assertNotEquals(null, scheduler); }
Example #24
Source File: AbstractHBaseClient.java From jstorm with Apache License 2.0 | 5 votes |
public Configuration makeConf(Map stormConf) { Configuration hbaseConf = HBaseConfiguration.create(); String hbaseQuorum = (String) stormConf.get(HBASE_QUORUM_CONF_KEY); hbaseConf.set(HBASE_QUORUM_KEY, hbaseQuorum); String hbasePort = stormConf.get(HBASE_PORT_CONF_KEY) + ""; hbaseConf.set(HBASE_PORT_KEY, hbasePort); String hbaseParent = (String) stormConf.get(HBASE_ZK_PARENT_CONF_KEY); hbaseConf.set(HBASE_ZK_PARENT_KEY, hbaseParent); return hbaseConf; }
Example #25
Source File: IndexUpgradeToolTest.java From phoenix with Apache License 2.0 | 5 votes |
@Test public void testConnectionProperties() throws Exception { Configuration conf = HBaseConfiguration.create(); long indexRebuildQueryTimeoutMs = 2000; long indexRebuildRpcTimeoutMs = 3000; long indexRebuildClientScannerTimeoutMs = 4000; int indexRebuildRpcRetryCount = 10; conf.setLong(QueryServices.INDEX_REBUILD_QUERY_TIMEOUT_ATTRIB, indexRebuildQueryTimeoutMs); conf.setLong(QueryServices.INDEX_REBUILD_RPC_TIMEOUT_ATTRIB, indexRebuildRpcTimeoutMs); conf.setLong(QueryServices.INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT_ATTRIB, indexRebuildClientScannerTimeoutMs); conf.setInt(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, indexRebuildRpcRetryCount); // prepare conf for connectionless query setupConfForConnectionlessQuery(conf); try (Connection conn = IndexUpgradeTool.getConnection(conf)) { // verify connection properties for phoenix, hbase timeouts and retries Assert.assertEquals(conn.getClientInfo(QueryServices.THREAD_TIMEOUT_MS_ATTRIB), Long.toString(indexRebuildQueryTimeoutMs)); Assert.assertEquals(conn.getClientInfo(HConstants.HBASE_RPC_TIMEOUT_KEY), Long.toString(indexRebuildRpcTimeoutMs)); Assert.assertEquals(conn.getClientInfo(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD), Long.toString(indexRebuildClientScannerTimeoutMs)); Assert.assertEquals(conn.getClientInfo(HConstants.HBASE_CLIENT_RETRIES_NUMBER), Long.toString(indexRebuildRpcRetryCount)); } }
Example #26
Source File: TestHeapMemoryManager.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testPluggingInHeapMemoryTuner() throws Exception { BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4)); MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4)); Configuration conf = HBaseConfiguration.create(); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f); conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0); conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class); // Let the system start with default values for memstore heap and block cache size. HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), new RegionServerAccountingStub(conf)); final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); heapMemoryManager.start(choreService); // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner CustomHeapMemoryTuner.memstoreSize = 0.78f; CustomHeapMemoryTuner.blockCacheSize = 0.02f; // Allow the tuner to run once and do necessary memory up waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner CustomHeapMemoryTuner.blockCacheSize = 0.75f; CustomHeapMemoryTuner.memstoreSize = 0.05f; // Allow the tuner to run once and do necessary memory up waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore }
Example #27
Source File: Mapper2HbaseDemo.java From bigdata-tutorial with Apache License 2.0 | 5 votes |
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); conf = HBaseConfiguration.create(context.getConfiguration()); conf.set("hbase.zookeeper.quorum", "zk1.hadoop,zk2.hadoop,zk3.hadoop"); conf.set("hbase.zookeeper.property.clientPort", "2181"); htable = new HTable(conf, "micmiu"); htable.setAutoFlush(false); htable.setWriteBufferSize(12 * 1024 * 1024);//12M wal = true; }
Example #28
Source File: TestZKUtilNoServer.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testCreateACLWithSameUser() throws IOException { Configuration conf = HBaseConfiguration.create(); conf.set(Superusers.SUPERUSER_CONF_KEY, "user4,@group1,user5,user6"); UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser("user4")); String node = "/hbase/testCreateACL"; ZKWatcher watcher = new ZKWatcher(conf, node, null, false); List<ACL> aclList = ZKUtil.createACL(watcher, node, true); assertEquals(3, aclList.size()); // 3, since service user the same as one of superuser assertFalse(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "@group1")))); assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("auth", "")))); assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user5")))); assertTrue(aclList.contains(new ACL(Perms.ALL, new Id("sasl", "user6")))); }
Example #29
Source File: Sandbox.java From phoenix with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { System.out.println("Starting Phoenix sandbox"); Configuration conf = HBaseConfiguration.create(); BaseTest.setUpConfigForMiniCluster(conf, new ReadOnlyProps(ImmutableMap.<String, String>of())); final HBaseTestingUtility testUtil = new HBaseTestingUtility(conf); testUtil.startMiniCluster(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { if (testUtil != null) { testUtil.shutdownMiniCluster(); } } catch (Exception e) { LOG.error("Exception caught when shutting down mini cluster", e); } } }); int clientPort = testUtil.getZkCluster().getClientPort(); System.out.println("\n\n\tPhoenix Sandbox is started\n\n"); System.out.printf("\tYou can now connect with url 'jdbc:phoenix:localhost:%d'\n" + "\tor connect via sqlline with 'bin/sqlline.py localhost:%d'\n\n", clientPort, clientPort); Thread.sleep(Long.MAX_VALUE); }
Example #30
Source File: TestStripeCompactionPolicy.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testNothingToCompactFromL0() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 4); StripeCompactionPolicy.StripeInformationProvider si = createStripesL0Only(3, 10); StripeCompactionPolicy policy = createPolicy(conf); verifyNoCompaction(policy, si); si = createStripes(3, KEY_A); verifyNoCompaction(policy, si); }