Java Code Examples for org.apache.hadoop.hbase.HBaseTestingUtility#getZkCluster()

The following examples show how to use org.apache.hadoop.hbase.HBaseTestingUtility#getZkCluster() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestReplicationStuckWithDeletedTableCFs.java From hbase with Apache License 2.0

6 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();
}

Example 2

Source File: TestMasterReplication.java From hbase with Apache License 2.0

6 votes

@SuppressWarnings("resource")
private void startMiniClusters(int numClusters) throws Exception {
  Random random = new Random();
  utilities = new HBaseTestingUtility[numClusters];
  configurations = new Configuration[numClusters];
  for (int i = 0; i < numClusters; i++) {
    Configuration conf = new Configuration(baseConfiguration);
    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/" + i + random.nextInt());
    HBaseTestingUtility utility = new HBaseTestingUtility(conf);
    if (i == 0) {
      utility.startMiniZKCluster();
      miniZK = utility.getZkCluster();
    } else {
      utility.setZkCluster(miniZK);
    }
    utility.startMiniCluster();
    utilities[i] = utility;
    configurations[i] = conf;
    new ZKWatcher(conf, "cluster" + i, null, true);
  }
}

Example 3

Source File: TestReplicationStuckWithDroppedTable.java From hbase with Apache License 2.0

6 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.set(ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();
}

Example 4

Source File: TestReplicationEditsDroppedWithDeletedTableCFs.java From hbase with Apache License 2.0

6 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Set true to filter replication edits for dropped table
  conf1.setBoolean(REPLICATION_DROP_ON_DELETED_COLUMN_FAMILY_KEY, true);
  conf1.set(ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();
}

Example 5

Source File: TestReplicationEditsDroppedWithDroppedTable.java From hbase with Apache License 2.0

5 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Set true to filter replication edits for dropped table
  conf1.setBoolean(HBaseInterClusterReplicationEndpoint.REPLICATION_DROP_ON_DELETED_TABLE_KEY,
      true);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();

  NamespaceDescriptor nsDesc = NamespaceDescriptor.create(namespace).build();
  admin1.createNamespace(nsDesc);
  admin2.createNamespace(nsDesc);
}

Example 6

Source File: TestGlobalReplicationThrottler.java From hbase with Apache License 2.0

5 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1 = HBaseConfiguration.create();
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setLong("replication.source.sleepforretries", 100);
  // Each WAL is about 120 bytes
  conf1.setInt(HConstants.REPLICATION_SOURCE_TOTAL_BUFFER_KEY, REPLICATION_SOURCE_QUOTA);
  conf1.setLong("replication.source.per.peer.node.bandwidth", 100L);

  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  new ZKWatcher(conf1, "cluster1", null, true);

  conf2 = new Configuration(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");

  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);
  new ZKWatcher(conf2, "cluster2", null, true);

  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(utility2.getClusterKey());

  utility1.startMiniCluster();
  utility2.startMiniCluster();

  Admin admin1 = ConnectionFactory.createConnection(conf1).getAdmin();
  admin1.addReplicationPeer("peer1", rpc);
  admin1.addReplicationPeer("peer2", rpc);
  admin1.addReplicationPeer("peer3", rpc);
  numOfPeer = admin1.listReplicationPeers().size();
}

Example 7

Source File: TestVerifyReplicationCrossDiffHdfs.java From hbase with Apache License 2.0

5 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  util1 = new HBaseTestingUtility(conf1);
  util1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = util1.getZkCluster();
  conf1 = util1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  util2 = new HBaseTestingUtility(conf2);
  util2.setZkCluster(miniZK);

  util1.startMiniCluster();
  util2.startMiniCluster();

  createTestingTable(util1.getAdmin());
  createTestingTable(util2.getAdmin());
  addTestingPeer();

  LOG.info("Start to load some data to source cluster.");
  loadSomeData();

  LOG.info("Start mini MapReduce cluster.");
  mapReduceUtil.setZkCluster(miniZK);
  mapReduceUtil.startMiniMapReduceCluster();
}

Example 8

Source File: MutableIndexReplicationIT.java From phoenix with Apache License 2.0

4 votes

private static void setupConfigsAndStartCluster() throws Exception {
        // cluster-1 lives at regular HBase home, so we don't need to change how phoenix handles
        // lookups
//        conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
        // smaller log roll size to trigger more events
        setUpConfigForMiniCluster(conf1);
        conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
        conf1.setInt("replication.source.size.capacity", 10240);
        conf1.setLong("replication.source.sleepforretries", 100);
        conf1.setInt("hbase.regionserver.maxlogs", 10);
        conf1.setLong("hbase.master.logcleaner.ttl", 10);
        conf1.setInt("zookeeper.recovery.retry", 1);
        conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
        conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
        conf1.setBoolean("dfs.support.append", true);
        conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
        conf1.setInt("replication.stats.thread.period.seconds", 5);
        conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);

        utility1 = new HBaseTestingUtility(conf1);
        utility1.startMiniZKCluster();
        MiniZooKeeperCluster miniZK = utility1.getZkCluster();
        // Have to reset conf1 in case zk cluster location different
        // than default
        conf1 = utility1.getConfiguration();
        zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
        admin = new ReplicationAdmin(conf1);
        LOG.info("Setup first Zk");

        // Base conf2 on conf1 so it gets the right zk cluster, and general cluster configs
        conf2 = HBaseConfiguration.create(conf1);
        conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
        conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
        conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
        conf2.setBoolean("dfs.support.append", true);
        conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);

        utility2 = new HBaseTestingUtility(conf2);
        utility2.setZkCluster(miniZK);
        zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);

        //replicate from cluster 1 -> cluster 2, but not back again
        admin.addPeer("1", utility2.getClusterKey());

        LOG.info("Setup second Zk");
        utility1.startMiniCluster(2);
        utility2.startMiniCluster(2);
    }

Example 9

Source File: TestPerTableCFReplication.java From hbase with Apache License 2.0

4 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1 = HBaseConfiguration.create();
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  // smaller block size and capacity to trigger more operations
  // and test them
  conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
  conf1.setInt("replication.source.size.capacity", 1024);
  conf1.setLong("replication.source.sleepforretries", 100);
  conf1.setInt("hbase.regionserver.maxlogs", 10);
  conf1.setLong("hbase.master.logcleaner.ttl", 10);
  conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");

  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  new ZKWatcher(conf1, "cluster1", null, true);

  conf2 = new Configuration(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");

  conf3 = new Configuration(conf1);
  conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");

  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);
  new ZKWatcher(conf2, "cluster3", null, true);

  utility3 = new HBaseTestingUtility(conf3);
  utility3.setZkCluster(miniZK);
  new ZKWatcher(conf3, "cluster3", null, true);

  table = new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(famName);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  table.setColumnFamily(familyDescriptor);
  familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(noRepfamName);
  table.setColumnFamily(familyDescriptor);

  tabA = new TableDescriptorBuilder.ModifyableTableDescriptor(tabAName);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f1Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabA.setColumnFamily(familyDescriptor);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f2Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabA.setColumnFamily(familyDescriptor);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f3Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabA.setColumnFamily(familyDescriptor);

  tabB = new TableDescriptorBuilder.ModifyableTableDescriptor(tabBName);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f1Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabB.setColumnFamily(familyDescriptor);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f2Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabB.setColumnFamily(familyDescriptor);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f3Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabB.setColumnFamily(familyDescriptor);

  tabC = new TableDescriptorBuilder.ModifyableTableDescriptor(tabCName);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f1Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabC.setColumnFamily(familyDescriptor);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f2Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabC.setColumnFamily(familyDescriptor);
  familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(f3Name);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tabC.setColumnFamily(familyDescriptor);

  utility1.startMiniCluster();
  utility2.startMiniCluster();
  utility3.startMiniCluster();
}

Example 10

Source File: TestMultiSlaveReplication.java From hbase with Apache License 2.0

4 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1 = HBaseConfiguration.create();
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  // smaller block size and capacity to trigger more operations
  // and test them
  conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
  conf1.setInt("replication.source.size.capacity", 1024);
  conf1.setLong("replication.source.sleepforretries", 100);
  conf1.setInt("hbase.regionserver.maxlogs", 10);
  conf1.setLong("hbase.master.logcleaner.ttl", 10);
  conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");
  conf1.setInt("hbase.master.cleaner.interval", 5 * 1000);

  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  utility1.setZkCluster(miniZK);
  new ZKWatcher(conf1, "cluster1", null, true);

  conf2 = new Configuration(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");

  conf3 = new Configuration(conf1);
  conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");

  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);
  new ZKWatcher(conf2, "cluster2", null, true);

  utility3 = new HBaseTestingUtility(conf3);
  utility3.setZkCluster(miniZK);
  new ZKWatcher(conf3, "cluster3", null, true);

  table = new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(famName);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  table.setColumnFamily(familyDescriptor);
  familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(noRepfamName);
  table.setColumnFamily(familyDescriptor);
}

Example 11

Source File: TestReplicationWithTags.java From hbase with Apache License 2.0

4 votes

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.setInt("hfile.format.version", 3);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.size.capacity", 10240);
  conf1.setLong("replication.source.sleepforretries", 100);
  conf1.setInt("hbase.regionserver.maxlogs", 10);
  conf1.setLong("hbase.master.logcleaner.ttl", 10);
  conf1.setInt("zookeeper.recovery.retry", 1);
  conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf1.setInt("replication.stats.thread.period.seconds", 5);
  conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessorForTagsAtSource.class.getName());

  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  // Have to reget conf1 in case zk cluster location different
  // than default
  conf1 = utility1.getConfiguration();
  LOG.info("Setup first Zk");

  // Base conf2 on conf1 so it gets the right zk cluster.
  conf2 = HBaseConfiguration.create(conf1);
  conf2.setInt("hfile.format.version", 3);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf2.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf2.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
          TestCoprocessorForTagsAtSink.class.getName());

  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  LOG.info("Setup second Zk");
  utility1.startMiniCluster(2);
  utility2.startMiniCluster(2);

  connection1 = ConnectionFactory.createConnection(conf1);
  replicationAdmin = connection1.getAdmin();
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(utility2.getClusterKey());
  replicationAdmin.addReplicationPeer("2", rpc);

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY);
  familyDescriptor.setMaxVersions(3);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tableDescriptor.setColumnFamily(familyDescriptor);
  try (Connection conn = ConnectionFactory.createConnection(conf1);
      Admin admin = conn.getAdmin()) {
    admin.createTable(tableDescriptor, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  try (Connection conn = ConnectionFactory.createConnection(conf2);
      Admin admin = conn.getAdmin()) {
    admin.createTable(tableDescriptor, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  htable1 = utility1.getConnection().getTable(TABLE_NAME);
  htable2 = utility2.getConnection().getTable(TABLE_NAME);
}

Example 12

Source File: TestVisibilityLabelsReplication.java From hbase with Apache License 2.0

4 votes

@Before
public void setup() throws Exception {
  // setup configuration
  conf = HBaseConfiguration.create();
  conf.setInt("hfile.format.version", 3);
  conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  setVisibilityLabelServiceImpl(conf);
  conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  VisibilityTestUtil.enableVisiblityLabels(conf);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      VisibilityReplication.class.getName());
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      SimpleCP.class.getName());
  // Have to reset conf1 in case zk cluster location different
  // than default
  conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
      ScanLabelGenerator.class);
  conf.set("hbase.superuser", User.getCurrent().getShortName());
  SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(),
      new String[] { "supergroup" });
  // User.createUserForTesting(conf, User.getCurrent().getShortName(), new
  // String[] { "supergroup" });
  USER1 = User.createUserForTesting(conf, "user1", new String[] {});
  TEST_UTIL = new HBaseTestingUtility(conf);
  TEST_UTIL.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
  zkw1 = new ZKWatcher(conf, "cluster1", null, true);

  // Base conf2 on conf1 so it gets the right zk cluster.
  conf1 = HBaseConfiguration.create(conf);
  conf1.setInt("hfile.format.version", 3);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessorForTagsAtSink.class.getName());
  // setVisibilityLabelServiceImpl(conf1);
  USER1 = User.createUserForTesting(conf1, "user1", new String[] {});
  TEST_UTIL1 = new HBaseTestingUtility(conf1);
  TEST_UTIL1.setZkCluster(miniZK);
  zkw2 = new ZKWatcher(conf1, "cluster2", null, true);

  TEST_UTIL.startMiniCluster(1);
  // Wait for the labels table to become available
  TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
  TEST_UTIL1.startMiniCluster(1);

  admin = TEST_UTIL.getAdmin();
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(TEST_UTIL1.getClusterKey());
  admin.addReplicationPeer("2", rpc);

  Admin hBaseAdmin = TEST_UTIL.getAdmin();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tableDescriptor.setColumnFamily(familyDescriptor);
  try {
    hBaseAdmin.createTable(tableDescriptor);
  } finally {
    if (hBaseAdmin != null) {
      hBaseAdmin.close();
    }
  }
  Admin hBaseAdmin1 = TEST_UTIL1.getAdmin();
  try {
    hBaseAdmin1.createTable(tableDescriptor);
  } finally {
    if (hBaseAdmin1 != null) {
      hBaseAdmin1.close();
    }
  }
  addLabels();
  setAuths(conf);
  setAuths(conf1);
}

Example 13

Source File: TestVisibilityLabelReplicationWithExpAsString.java From hbase with Apache License 2.0

4 votes

@Override
@Before
public void setup() throws Exception {
  expected[0] = 4;
  expected[1] = 6;
  expected[2] = 4;
  expected[3] = 0;
  expected[3] = 3;
  expectedVisString[0] = "(\"public\"&\"secret\"&\"topsecret\")|(\"confidential\"&\"topsecret\")";
  expectedVisString[1] = "(\"private\"&\"public\")|(\"private\"&\"topsecret\")|"
      + "(\"confidential\"&\"public\")|(\"confidential\"&\"topsecret\")";
  expectedVisString[2] = "(!\"topsecret\"&\"secret\")|(!\"topsecret\"&\"confidential\")";
  expectedVisString[3] = "(\"secret\"&\"" + COPYRIGHT + "\\\"" + ACCENT + "\\\\" + SECRET
      + "\\\"" + "\u0027&\\\\" + "\")";
  // setup configuration
  conf = HBaseConfiguration.create();
  conf.setInt("hfile.format.version", 3);
  conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  setVisibilityLabelServiceImpl(conf, ExpAsStringVisibilityLabelServiceImpl.class);
  conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  VisibilityTestUtil.enableVisiblityLabels(conf);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      VisibilityReplication.class.getName());
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      SimpleCP.class.getName());
  // Have to reset conf1 in case zk cluster location different
  // than default
  conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
          ScanLabelGenerator.class);
  conf.set("hbase.superuser", "admin");
  conf.set("hbase.superuser", User.getCurrent().getShortName());
  SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(),
      new String[] { "supergroup" });
  User.createUserForTesting(conf,
      User.getCurrent().getShortName(), new String[] { "supergroup" });
  USER1 = User.createUserForTesting(conf, "user1", new String[] {});
  TEST_UTIL = new HBaseTestingUtility(conf);
  TEST_UTIL.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
  zkw1 = new ZKWatcher(conf, "cluster1", null, true);

  // Base conf2 on conf1 so it gets the right zk cluster.
  conf1 = HBaseConfiguration.create(conf);
  conf1.setInt("hfile.format.version", 3);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
          TestCoprocessorForTagsAtSink.class.getName());
  setVisibilityLabelServiceImpl(conf1, ExpAsStringVisibilityLabelServiceImpl.class);
  TEST_UTIL1 = new HBaseTestingUtility(conf1);
  TEST_UTIL1.setZkCluster(miniZK);
  zkw2 = new ZKWatcher(conf1, "cluster2", null, true);

  TEST_UTIL.startMiniCluster(1);
  // Wait for the labels table to become available
  TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
  TEST_UTIL1.startMiniCluster(1);

  admin = TEST_UTIL.getAdmin();
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(TEST_UTIL1.getClusterKey());
  admin.addReplicationPeer("2", rpc);

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tableDescriptor.setColumnFamily(familyDescriptor);
  try (Admin hBaseAdmin = TEST_UTIL.getAdmin()) {
    hBaseAdmin.createTable(tableDescriptor);
  }
  try (Admin hBaseAdmin1 = TEST_UTIL1.getAdmin()){
    hBaseAdmin1.createTable(tableDescriptor);
  }
  addLabels();
  setAuths(conf);
  setAuths(conf1);
}

Example 14

Source File: MutableIndexReplicationIT.java From phoenix with Apache License 2.0

4 votes

private static void setupConfigsAndStartCluster() throws Exception {
        // cluster-1 lives at regular HBase home, so we don't need to change how phoenix handles
        // lookups
//        conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
        // smaller log roll size to trigger more events
        setUpConfigForMiniCluster(conf1);
        conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
        conf1.setInt("replication.source.size.capacity", 10240);
        conf1.setLong("replication.source.sleepforretries", 100);
        conf1.setInt("hbase.regionserver.maxlogs", 10);
        conf1.setLong("hbase.master.logcleaner.ttl", 10);
        conf1.setInt("zookeeper.recovery.retry", 1);
        conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
        conf1.setBoolean("dfs.support.append", true);
        conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
        conf1.setInt("replication.stats.thread.period.seconds", 5);
        conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);

        utility1 = new HBaseTestingUtility(conf1);
        utility1.startMiniZKCluster();
        MiniZooKeeperCluster miniZK = utility1.getZkCluster();
        // Have to reset conf1 in case zk cluster location different
        // than default
        conf1 = utility1.getConfiguration();
        zkw1 = new ZKWatcher(conf1, "cluster1", null, true);
        admin=ConnectionFactory.createConnection(conf1).getAdmin();
        LOGGER.info("Setup first Zk");

        // Base conf2 on conf1 so it gets the right zk cluster, and general cluster configs
        conf2 = HBaseConfiguration.create(conf1);
        conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
        conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
        conf2.setBoolean("dfs.support.append", true);
        conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);

        utility2 = new HBaseTestingUtility(conf2);
        utility2.setZkCluster(miniZK);
        zkw2 = new ZKWatcher(conf2, "cluster2", null, true);

        LOGGER.info("Setup second Zk");
        utility1.startMiniCluster(2);
        utility2.startMiniCluster(2);
      //replicate from cluster 1 -> cluster 2, but not back again
        admin.addReplicationPeer("1", new ReplicationPeerConfig().setClusterKey(utility2.getClusterKey()));
    }